{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport gc\n\nimport os\ntmp_dir='../input'\nprint(os.listdir(tmp_dir))\n\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4835886dfec930d6f8a3349b594cf8ef6e5ab454"},"cell_type":"markdown","source":"# train and test"},{"metadata":{"_uuid":"f5f9e6e851405bdff9ff5e98f88f6831f334bdea"},"cell_type":"markdown","source":"## describe"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv(tmp_dir+'/train.csv')\ntest=pd.read_csv(tmp_dir+'/test.csv')\ncol_describe=pd.read_excel(tmp_dir+'/Data_Dictionary.xlsx')\nprint(train.shape)\nprint(test.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3797ceae9a519ef6f237841dd2f458e02cdd70be","trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a3a2c3404b65c1f63ca3e320a04d3fe2fb61882","trusted":true},"cell_type":"code","source":"test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5549aaf19bb666d597fda85cc1e9617933103a8","trusted":true},"cell_type":"code","source":"for i in [ 'feature_1', 'feature_2', 'feature_3']:\n    train[i]=train[i].astype('category')\n    test[i]=test[i].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eed9856a76a15c3a908464c1aeea89fc9754622b","trusted":true},"cell_type":"code","source":"train.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67871c2a844aaa941b4d9933b326111c520f9b5b","trusted":true},"cell_type":"code","source":"test.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a139295ba0994f5fefdb187754c63a60c9a037c"},"cell_type":"markdown","source":"## target"},{"metadata":{"_uuid":"a482177909393b4e9a395e51ec3bfa6978856520","trusted":true},"cell_type":"code","source":"sns.distplot(train.target)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"9e1fbd32000c2306de9e6afa7ba4aadda0b65efb","trusted":true},"cell_type":"code","source":"train.target.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dea973b1ac91d3a8795797f4aa224f0e7b0e6d14"},"cell_type":"markdown","source":"### abnormal\n\nit seems that -33.219281 is abnormal,I remove these data"},{"metadata":{"scrolled":true,"_uuid":"8af8a97df2c919740a985f7185e2216e81725976","trusted":true},"cell_type":"code","source":"train['target2']=train.target.map(lambda x:round(x,3))\ntrain=train[train.target2!=-33.219]\ndel train['target2']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e3a72989541e891cd6accfac998dc55b3337d60"},"cell_type":"markdown","source":"## first_active_month"},{"metadata":{"scrolled":true,"_uuid":"13cf174f665d4d63f45092f0373f45265160c135","trusted":true},"cell_type":"code","source":"### fill null \na=test.first_active_month.value_counts()\ntest.first_active_month.fillna(a[a.values==a.max()].index[0],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4aa65c9b2d6456064bb7c49839c2053cdc9d1fc4","trusted":true},"cell_type":"code","source":"print(train.first_active_month.min())\nprint(train.first_active_month.max())\nprint(test.first_active_month.min())\nprint(test.first_active_month.max())\n\ntrain.first_active_month.describe()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e84dfee7e853ffc365fdccffec5295ed2aa6e732","trusted":true},"cell_type":"code","source":"## count by first_active_month\ntest_month_count=test.first_active_month.value_counts().sort_index()\ntrain_month_count=train.first_active_month.value_counts().sort_index()\n\nplt.plot(test_month_count.index,test_month_count.values)\nplt.plot(train_month_count.index,train_month_count.values)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"_uuid":"7bc626e67cb4bbc380e62a1552d694899103d17d","trusted":true},"cell_type":"code","source":"## mean of target by first_active_month\n\nmean_target=train.groupby('first_active_month')['target'].mean().sort_index()\nplt.plot(mean_target.index,mean_target.values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0beebbc2a5e88b2c29cc5a12336aef231112d3d1"},"cell_type":"markdown","source":"\n## FeatureEngin "},{"metadata":{"_uuid":"572be0655c5abcc434086c1211ff10ef4dc5b80a","trusted":true},"cell_type":"code","source":"y_true=train.target\ndel train['target']\ntrain_count=train.shape[0]   \ndf=pd.concat([train,test],axis=0) ##train=df.iloc[:train_count]\ndel train\ndel test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd289ef0a8863a75be3b1de067c0f3a6dfb111a7","trusted":true},"cell_type":"code","source":"df.first_active_month=df.first_active_month.astype('category')\ndf_dummy=pd.get_dummies(df[['feature_1','feature_2','feature_3','first_active_month']])\ndf=pd.concat([df,df_dummy],axis=1)\ndel df_dummy\n\ndf['first_active_month']=pd.to_datetime(df['first_active_month'])\ndf['first_month']=df.first_active_month.dt.month\nmax_date=df.first_active_month.dt.date.max()\ndf['lantency']=max_date - df.first_active_month.dt.date\ndf['lantency']=df.lantency.dt.days\n\ndf=df.drop(['feature_1', 'feature_2','feature_3','first_active_month'], axis=1)\ndf.first_month=df.first_month.astype('int8')\ndf.lantency=df.lantency.astype('int8')\ngc.collect()\ndf.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"747d78e57869dc66a945c13424f70a44ba4970e9"},"cell_type":"markdown","source":"#  new_merchant_transactions"},{"metadata":{"_uuid":"4d240e9b7bd37940434a8e30656983d86e8d4ccd"},"cell_type":"markdown","source":"## merchants"},{"metadata":{"_uuid":"8f57293d3e506ef77aa199a23c9a21421bf85187","trusted":true},"cell_type":"code","source":"### merchants.csv - additional information about all merchants / merchant_ids in the dataset.\n# merchants=pd.read_csv(tmp_dir+'/merchants.csv')\n# print(merchants.shape)\n# merchants.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31439c781f00a8a82675faa4e632f7f06e1c4d8d","trusted":true},"cell_type":"code","source":"# a=merchants.merchant_id.value_counts()\n# merchants2=merchants.groupby('merchant_id').head(1)\n# merchants2.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6894654986a4ca272fddbabe5b3c460f6f349216"},"cell_type":"markdown","source":"##  new_merchant_transactions"},{"metadata":{"_uuid":"68587a6a16dae3874c0153aca75e14c02d3f8577","trusted":true},"cell_type":"code","source":"##new_merchant_transactions.csv - two months' worth of data for each card_id \n###containing ALL purchases that card_id made at merchant_ids that were not visited in the historical data.\n\nnew_merchant_transactions=pd.read_csv(tmp_dir+'/new_merchant_transactions.csv')\nprint(new_merchant_transactions.shape)\nfor i in [ 'city_id','category_1','merchant_category_id','category_2','state_id',\n          'subsector_id','authorized_flag','category_3','merchant_id']:\n    new_merchant_transactions[i]=new_merchant_transactions[i].astype('category')\nnew_merchant_transactions.month_lag=new_merchant_transactions.month_lag.astype('int8')\nnew_merchant_transactions.installments=new_merchant_transactions.installments.astype('int8')\nnew_merchant_transactions.purchase_amount=new_merchant_transactions.purchase_amount.astype('float32')\n\nprint(new_merchant_transactions.dtypes)\nnew_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b16de79f95550b0fd133a7b43f63a62ca14d11f8","trusted":true},"cell_type":"code","source":"# new_merchant_transactions.head()\n# new_merchant_transactions.describe(include='category')\n# new_merchant_transactions.describe()\n# new_merchant_transactions.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03a749486a15dd90db5086ada8e1440491a8e329","trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\nle=preprocessing.LabelEncoder()\nle.fit(new_merchant_transactions['category_1'])\nnew_merchant_transactions['category_1']=le.transform(new_merchant_transactions['category_1'])\nnew_merchant_transactions['category_1']=new_merchant_transactions['category_1'].astype('int8')\nnew_merchant_transactions['purchase_amount']=new_merchant_transactions['purchase_amount'].astype('float32')\n\n \ntmp=pd.get_dummies(new_merchant_transactions[['category_2','category_3']])\nnew_merchant_transactions=pd.concat([new_merchant_transactions,tmp],axis=1)\n\nnew_merchant_transactions.purchase_date=pd.to_datetime(new_merchant_transactions.purchase_date)\nfor i in ['hour','day','dayofweek','month']:\n    new_merchant_transactions['purchase_'+i]=getattr(new_merchant_transactions.purchase_date.dt,i)\n    new_merchant_transactions['purchase_'+i]=new_merchant_transactions['purchase_'+i].astype('int8')\nnew_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"run_control":{"marked":true},"_uuid":"410a8df1722cb5b20161903b4ac9ab46404bb6f9","trusted":true},"cell_type":"code","source":"%%time\n    \nagg_fun={'category_1': ['sum', 'mean'],\n'category_2_1.0': ['sum', 'mean'],\n'category_2_2.0': ['sum', 'mean'],\n'category_2_3.0': ['sum', 'mean'],\n'category_2_4.0': ['sum', 'mean'],\n'category_2_5.0': ['sum', 'mean'],\n'category_3_A': ['sum', 'mean'],\n'category_3_B': ['sum', 'mean'],\n'category_3_C': ['sum', 'mean'],\n'month_lag': ['sum', 'mean'],\n'installments': ['sum', 'mean','min','max','std'],\n'purchase_amount': ['sum', 'mean','min','max','std'],\n        \n'state_id':['nunique'], \n'city_id': ['nunique'],\n'merchant_category_id': ['nunique'],\n'subsector_id': ['nunique'],\n'merchant_id': ['nunique'],\n         \n'purchase_hour': ['min', 'max'],\n'purchase_day': ['min', 'max'],\n'purchase_dayofweek': ['min', 'max'],\n'purchase_month': ['min', 'max'],\n\n'card_id':'count',\n}\n\nnew_agg=new_merchant_transactions.groupby('card_id').agg(agg_fun)\n\nnew_agg.columns=['new_' + '_'.join(col).strip() for col in new_agg.columns.values]\ndel new_merchant_transactions\ngc.collect()\n# new_agg.to_csv('new_agg.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6995ea4eb2dd5b5837aaf207f4dee2ce946fa4f6"},"cell_type":"markdown","source":"# history"},{"metadata":{"_uuid":"9479c1252053dca6d0431dbb393c89f5bbe24d3b","trusted":true},"cell_type":"code","source":"%%time\n#### historical_transactions.csv - up to 3 months' worth of historical transactions for each card_id\n\nhistorical_transactions=pd.read_csv(tmp_dir+'/historical_transactions.csv')\nprint(historical_transactions.shape)\nfor i in [ 'city_id','category_1','merchant_category_id','category_2','state_id',\n          'subsector_id','authorized_flag','category_3','merchant_id']:\n    historical_transactions[i]=historical_transactions[i].astype('category')\nhistorical_transactions['month_lag']=historical_transactions['month_lag'].astype('int8')\nhistorical_transactions.installments=historical_transactions.installments.astype('int8')\nhistorical_transactions.purchase_amount=historical_transactions.purchase_amount.astype('float32')\nprint(historical_transactions.dtypes)\nhistorical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aeae6f08d628f9e09473eb800db50303f7aa817f","trusted":true},"cell_type":"code","source":"le=preprocessing.LabelEncoder()\nle.fit(historical_transactions['category_1'])\nhistorical_transactions['category_1']=le.transform(historical_transactions['category_1'])\n\ntmp=pd.get_dummies(historical_transactions[['category_2','category_3']])\nhistorical_transactions=pd.concat([historical_transactions,tmp],axis=1)\n\nhistorical_transactions.purchase_date=pd.to_datetime(historical_transactions.purchase_date)\nfor i in ['hour','day','dayofweek','month']:\n    historical_transactions['purchase_'+i]=getattr(historical_transactions.purchase_date.dt,i)\n    historical_transactions['purchase_'+i]=historical_transactions['purchase_'+i].astype('int8')\nhistorical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27ee1911dfaccac90ecb422a96f6813ddd325fd0","trusted":true},"cell_type":"code","source":"%%time\n    \nagg_fun={'category_1': ['sum', 'mean'],\n'category_2_1.0': ['sum', 'mean'],\n'category_2_2.0': ['sum', 'mean'],\n'category_2_3.0': ['sum', 'mean'],\n'category_2_4.0': ['sum', 'mean'],\n'category_2_5.0': ['sum', 'mean'],\n'category_3_A': ['sum', 'mean'],\n'category_3_B': ['sum', 'mean'],\n'category_3_C': ['sum', 'mean'],\n'month_lag': ['sum', 'mean'],\n'installments': ['sum', 'mean','min','max','std'],\n'purchase_amount': ['sum', 'mean','min','max','std'],\n        \n'state_id':['nunique'], \n'city_id': ['nunique'],\n'merchant_category_id': ['nunique'],\n'subsector_id': ['nunique'],\n'merchant_id': ['nunique'],\n         \n'purchase_hour': ['min', 'max'],\n'purchase_day': ['min', 'max'],\n'purchase_dayofweek': ['min', 'max'],\n'purchase_month': ['min', 'max'],\n\n'card_id':'count',\n}\n\nhis_agg=historical_transactions.groupby('card_id').agg(agg_fun)\nhis_agg.columns=['his_' + '_'.join(col).strip() for col in his_agg.columns.values]\ndel historical_transactions\ngc.collect()\n# his_agg.to_csv('new_agg.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae2113e3ba83888e7729c641ff8e879aac8d3d54"},"cell_type":"markdown","source":"# baseline"},{"metadata":{"_uuid":"5824873d74bfd1375235fecd3076661c674416d0","trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e702a009c700af320d02c261fc8eeb5e79914e3","trusted":true},"cell_type":"code","source":"df=pd.merge(df,new_agg,left_on='card_id',right_index=True,how='left')\nprint(len(df.columns))\ndf=pd.merge(df,his_agg,left_on='card_id',right_index=True,how='left')\nprint(len(df.columns))\n\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\n\ndf=reduce_mem_usage(df)\ndf=df.set_index('card_id')\n\ndel new_agg\ndel his_agg\ndf_train=df.iloc[:train_count]\ndf_test=df.iloc[train_count:]\nprint('df_train.shape',df_train.shape,'df_test.shape',df_test.shape)\ndel df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cae6baf29004d7fbd4d1afb0edcc300a92915e87","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test =train_test_split(df_train, y_true, test_size=0.2, random_state=8)\n# del df_train","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"262dde7051a7dba337d382246827e6c1bc145179","trusted":true},"cell_type":"code","source":"%%time\n## trian by raw lightgbm\nfrom sklearn.metrics import mean_squared_error\ntrain_columns=X_train.columns.tolist()\ntrain_data = lgb.Dataset(X_train, label=y_train,feature_name=train_columns\n                        ,free_raw_data=False)\n\nparam = {'boosting_type':['gbdt'],'max_leaves':[33], 'min_data_in_leaf':[30],'max_depth':[5],\n         'objective':['regression'],'random_state':[8],'metric':['l2']}\nnum_round=100\nbst=lgb.train(param,train_data,num_round)\nmean_squared_error(y_test,bst.predict(X_test))\n","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"76a21b29477ec0a9e98029412a2db74408e18852","trusted":true},"cell_type":"code","source":"## train by sklearn GridSearchCV\n\n# from sklearn.metrics import mean_squared_error\n\n\n# param = {'boosting_type':['gbdt'],'max_leaves':[33], 'min_data_in_leaf':[30,40,50],'max_depth':[4,5,6],\n#          'objective':['regression'],'random_state':[8],'metric':['l2']}\n\n# clf=GridSearchCV(lgb.LGBMRegressor(),param_grid=param,cv=5,n_jobs=4,verbose=2)\n# clf.fit(X_train, y_train,feature_name=['first_active_month', 'feature_1', 'feature_2', 'feature_3'],\n#         categorical_feature=['first_active_month', 'feature_1', 'feature_2', 'feature_3'])\n\n# mean_squared_error(y_test,clf.best_estimator_.predict(X_test))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3adeb93b31b0bd671339f9444dd0cd018c6b6201"},"cell_type":"markdown","source":"## predict"},{"metadata":{"trusted":true,"_uuid":"8f9c56c5f9b87977ba3d4240dbbf5fea18f86bf4"},"cell_type":"code","source":"df_test.head()\n## predict by raw lightgbm\npredict_target=bst.predict(df_test)\n\n## predict  sklearn GridSearchCV\n# predict_target=clf.best_estimator_.predict(X_predict)\n\ndf_test['target']=predict_target\ndf_test=df_test.reset_index()\ndf_test[['card_id','target']].to_csv('submission.csv',index=False)\nprint('success asv to csv submission.csv')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"toc":{"nav_menu":{},"toc_window_display":true,"toc_section_display":true,"number_sections":true,"base_numbering":1,"title_cell":"Table of Contents","sideBar":true,"title_sidebar":"Contents","skip_h1_title":false,"toc_position":{"width":"426.667px","top":"150px","left":"10px","height":"calc(100% - 180px)"},"toc_cell":false},"varInspector":{"kernels_config":{"python":{"delete_cmd_postfix":"","library":"var_list.py","delete_cmd_prefix":"del ","varRefreshCmd":"print(var_dic_list())"},"r":{"delete_cmd_postfix":") ","library":"var_list.r","delete_cmd_prefix":"rm(","varRefreshCmd":"cat(var_dic_list()) "}},"cols":{"lenType":16,"lenName":16,"lenVar":40},"types_to_exclude":["module","function","builtin_function_or_method","instance","_Feature"],"window_display":false},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}