{"cells":[{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T15:57:14.066748Z","start_time":"2018-12-13T15:57:10.939762Z"},"trusted":true,"_uuid":"148bf1df9c55a5731bb81a2282805afdade86152"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport gc\nimport os\nimport sys\nimport datetime\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm_notebook\nfrom scipy.sparse import hstack,csr_matrix\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\nfrom collections import Counter\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom joblib import Parallel,delayed\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error,mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.feature_selection import SelectPercentile\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import OneHotEncoder,StandardScaler,LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T15:57:14.395869Z","start_time":"2018-12-13T15:57:14.169474Z"},"trusted":true,"_uuid":"b25119381cd446e17ee73a3e7d63e101abe3beb2"},"cell_type":"code","source":"def datetime_transform(df):\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['year'] = pd.to_datetime(df['first_active_month']).dt.year\n    df['month'] = pd.to_datetime(df['first_active_month']).dt.month\n    return df\n\n\ndef applyParallel(dfGrouped, func):\n    retLst = Parallel(n_jobs=2)(delayed(func)(group)\n                                for name, group in dfGrouped)\n    return retLst\n\n\ndef get_stat_feature(df):\n    city_cnt = df['city_id'].nunique()\n    mer_cnt = df['merchant_id'].nunique()\n    state_cnt = df['state_id'].nunique()\n    installments = len(df['installments'] > 0)\n    return [df['card_id'].values[0], city_cnt, mer_cnt, state_cnt, installments]","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"f16de7ca3ccd6b79aeda5e8469a74e978c1414ca"},"cell_type":"markdown","source":"load data file"},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T15:59:44.030017Z","start_time":"2018-12-13T15:57:14.467681Z"},"trusted":true,"_uuid":"44aaffd2d6e98ef5ad2d171f2b8f44cbcff91068"},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv',)\ntest = pd.read_csv('../input/test.csv')\nht = pd.read_csv('../input/historical_transactions.csv')\nnmt = pd.read_csv('../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:00:07.009562Z","start_time":"2018-12-13T15:59:44.793663Z"},"trusted":true,"_uuid":"32d1996c29e9dc13f7a0fda473050ce1a1725274"},"cell_type":"code","source":"#日期变换\ntrain = datetime_transform(train)\ntest = datetime_transform(test)\nht['purchase_date'] = pd.to_datetime(ht['purchase_date'])\nnmt['purchase_date'] = pd.to_datetime(nmt['purchase_date'])\nht['purchase_year'] = ht['purchase_date'].dt.year\nnmt['purchase_year'] = nmt['purchase_date'].dt.year\nht['purchase_month'] = ht['purchase_date'].dt.month\nnmt['purchase_month'] = nmt['purchase_date'].dt.month","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:00:32.247916Z","start_time":"2018-12-13T16:00:07.566077Z"},"trusted":true,"_uuid":"24bfc3cb8f9a97f72653dd543a7f01d686093cec"},"cell_type":"code","source":"#缺失值填充\nht['category_2'] = ht['category_2'].fillna(0)\nht['category_3'] = ht['category_3'].fillna('E')\nnmt['category_2'] = nmt['category_2'].fillna(0)\nnmt['category_3'] = nmt['category_3'].fillna('E')\nht['category_2'] = ht['category_2'].map(lambda x:str(int(x)))\nnmt['category_2'] = nmt['category_2'].map(lambda x:str(int(x)))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:01:00.425588Z","start_time":"2018-12-13T16:00:32.685309Z"},"trusted":true,"_uuid":"1e5e68819f99cc4a5c775392cc414857b5ca8f26"},"cell_type":"code","source":"#数据类型转换\nht['merchant_category_id'] = ht['merchant_category_id'].map(lambda x:str(x))\nht['merchant_id'] = ht['merchant_id'].map(lambda x:str(x))\nnmt['merchant_category_id'] = nmt['merchant_category_id'].map(lambda x:str(x))\nnmt['merchant_id'] = nmt['merchant_id'].map(lambda x:str(x))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:01:40.187764Z","start_time":"2018-12-13T16:01:00.848937Z"},"trusted":true,"_uuid":"842e3f700108e7598580440a88d48cc649bee34d"},"cell_type":"code","source":"#部分数据转换，节省空间\nlb = LabelEncoder()\nht['authorized_flag'] = lb.fit_transform(ht['authorized_flag'])\nlb = LabelEncoder()\nht['category_1'] = lb.fit_transform(ht['category_1'])\nht['category_2'] = ht['category_2'].astype(np.int16)\nlb = LabelEncoder()\nht['category_3'] = lb.fit_transform(ht['category_3'])\nht['purchase_amount'] = ht['purchase_amount'].astype(np.float32)\nht['month_lag'] = ht['month_lag'].astype(np.float32)\nht['installments'] = ht['installments'].astype(np.int16)\nht['city_id'] = ht['city_id'].astype(np.int16)\nlb = LabelEncoder()\nht['merchant_category_id'] = lb.fit_transform(ht['merchant_category_id'])\nht['state_id'] = ht['state_id'].astype(np.int16)\nht['subsector_id'] = ht['subsector_id'].astype(np.int16)\nht['purchase_year'] = ht['purchase_year'].astype(np.int16)\nht['purchase_month'] = ht['purchase_month'].astype(np.int16)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b666edf181b3dc909d6197356498afa643bf513"},"cell_type":"markdown","source":"Start Feature Engineering"},{"metadata":{"trusted":true,"_uuid":"9fd616d2fa21224bee3fcedbf8f4bc5e2784b7c0"},"cell_type":"code","source":"v = pd.pivot_table(ht,values=['purchase_year'],index='card_id',columns=['state_id'],aggfunc=len,fill_value=0)\nv.columns = ['state_id_%s'%s for s in range(v.shape[1])]\nv = v.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f8693a449b5ff944a53e1cd735faa31df9cb82c"},"cell_type":"code","source":"train = pd.merge(train,v,on='card_id',how='left')\ntest = pd.merge(test,v,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:01:50.699145Z","start_time":"2018-12-13T16:01:40.636998Z"},"trusted":true,"_uuid":"b8e339d65091ba5869ce2737415ee4ba7eb21c88"},"cell_type":"code","source":"#激活日期距离今天的时间\ntrain['active_till_now'] = train['first_active_month'].map(lambda x:(datetime.datetime.today() - x).days)\ntest['active_till_now'] = test['first_active_month'].map(lambda x:(datetime.datetime.today() - x).days)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:02:00.100704Z","start_time":"2018-12-13T16:01:51.063176Z"},"trusted":true,"_uuid":"ea50635907fa96b7d63d3386a22bc3f0134c7248"},"cell_type":"code","source":"#统计历史上和在新商品上买过多少次东西\nht_cnt = ht.groupby('card_id')['city_id'].count().reset_index()\nnmt_cnt = nmt.groupby('card_id')['city_id'].count().reset_index()\nht_cnt.columns = ['card_id','ht_cnt']\nnmt_cnt.columns = ['card_id','nmt_cnt']\n#merge the result\ntrain = pd.merge(train,ht_cnt,on='card_id',how='left')\ntest = pd.merge(test,ht_cnt,on='card_id',how='left')\ntrain = pd.merge(train,nmt_cnt,on='card_id',how='left')\ntest = pd.merge(test,nmt_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82a74463ca67c5f730fa7024284b437accc18376"},"cell_type":"code","source":"ht_amount = ht.groupby('card_id').agg({'purchase_amount':[np.max,np.mean,np.std]})\nht_amount.columns = ['amount_max','amount_mean','amount_std']\nht_amount = ht_amount.reset_index()\n#计算历史上的消费状况\ntrain = pd.merge(train,ht_amount,on='card_id',how='left')\ntest = pd.merge(test,ht_amount,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bf128a3b972572c413328de0e1ecb897b3affee"},"cell_type":"code","source":"#计算历史上month_lag的情况\nht_month_lag = ht.groupby('card_id').agg({'month_lag':[np.min,np.mean,np.std]})\nht_month_lag.columns = ['lag_min','lag_mean','lag_std']\nht_month_lag = ht_month_lag.reset_index()\ntrain = pd.merge(train,ht_month_lag,on='card_id',how='left')\ntest = pd.merge(test,ht_month_lag,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:03:03.498767Z","start_time":"2018-12-13T16:02:40.618914Z"},"trusted":true,"_uuid":"b94183f6907949900810169684e61fc73b525256"},"cell_type":"code","source":"#计算历史上installments的情况\nht_installment = ht.groupby('card_id').agg({'installments':[np.max,np.min,np.mean,np.std]})\nht_installment.columns = ['ins_max','ins_min','ins_mean','ins_std']\nht_installment = ht_installment.reset_index()\ntrain = pd.merge(train,ht_installment,on='card_id',how='left')\ntest = pd.merge(test,ht_installment,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"470260abbec11ec7601e608734292475715c4b8d"},"cell_type":"code","source":"#计算新商店的消费状况\nnmt_amount = nmt.groupby('card_id').agg({'purchase_amount':[np.max,np.mean,np.std]})\nnmt_amount.columns = ['nmt_amount_max','nmt_amount_mean','nmt_amount_std']\nnmt_amount = nmt_amount.reset_index()\ntrain = pd.merge(train,nmt_amount,on='card_id',how='left')\ntest = pd.merge(test,nmt_amount,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b150a7e51f38aa8e2107b9c3710e863690f5f39f"},"cell_type":"code","source":"#计算新商店上month_lag的情况\nnmt_month_lag = nmt.groupby('card_id').agg({'month_lag':[np.max,np.mean,np.std]})\nnmt_month_lag.columns = ['nmt_lag_min','nmt_lag_mean','nmt_lag_std']\nnmt_month_lag = nmt_month_lag.reset_index()\ntrain = pd.merge(train,nmt_month_lag,on='card_id',how='left')\ntest = pd.merge(test,nmt_month_lag,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb809488cb36d1f5e6c16d0224c65ac42c769c59"},"cell_type":"code","source":"#计算新商店上installments的情况\nnmt_installment = nmt.groupby('card_id').agg({'installments':[np.max,np.mean,np.std]})\nnmt_installment.columns = ['nmt_ins_min','nmt_ins_mean','nmt_ins_std']\nnmt_installment = nmt_installment.reset_index()\ntrain = pd.merge(train,nmt_installment,on='card_id',how='left')\ntest = pd.merge(test,nmt_installment,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:19:53.393412Z","start_time":"2018-12-13T16:07:52.857242Z"},"trusted":true,"_uuid":"fe1b0b956d0a7188418c617e01c034599df4b683"},"cell_type":"code","source":"%%time\nht_res = applyParallel(ht.groupby('card_id'),get_stat_feature)\nnmt_res = applyParallel(nmt.groupby('card_id'),get_stat_feature)\n#merge the result\nstat_df = pd.DataFrame(ht_res,columns=['card_id','city_cnt','mer_cnt','state_cnt','installments'])\nnmt_stat_df = pd.DataFrame(nmt_res,columns=['card_id','city_cnt_nmt','mer_cnt_nmt','state_cnt_nmt','installments_num'])\ntrain = pd.merge(train,stat_df,on='card_id',how='left')\ntest = pd.merge(test,stat_df,on='card_id',how='left')\ntrain = pd.merge(train,nmt_stat_df,on='card_id',how='left')\ntest = pd.merge(test,nmt_stat_df,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"1f27676881b8791cde216b13c862b912f3ccbb4f"},"cell_type":"markdown","source":"拼接特征"},{"metadata":{"trusted":true,"_uuid":"beb11f63a2e3f495072c4f406f3a5d840a375757"},"cell_type":"code","source":"train.fillna(0,inplace=True)\ntest.fillna(0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:05.311601Z","start_time":"2018-12-13T16:07:05.304619Z"},"trusted":true,"_uuid":"20d25dc6288f43edb3c04e7c868092b7c6034067"},"cell_type":"code","source":"#handler the numeric feature\ndrop_columns = ['first_active_month','card_id','feature_1','feature_2','feature_3','target','year','month']\nnum_feature = [v for v in train.columns if v not in drop_columns]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:08.507123Z","start_time":"2018-12-13T16:07:07.438186Z"},"trusted":true,"_uuid":"995928a6b3d759a58d0ab9b8aa2a14d9a087688e"},"cell_type":"code","source":"train_num_feature = train[num_feature].values\ntest_num_feature = test[num_feature].values\nss = StandardScaler()\ntrain_num_feature = ss.fit_transform(train_num_feature)\ntest_num_feature = ss.transform(test_num_feature)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:16.726010Z","start_time":"2018-12-13T16:07:16.083803Z"},"trusted":true,"_uuid":"f28da062057df1fe90c6ce32a7c87490d353a8a9"},"cell_type":"code","source":"#stack the feature\ntrain_feature = train_num_feature\ntest_feature = test_num_feature","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:40.589836Z","start_time":"2018-12-13T16:07:19.022591Z"},"trusted":true,"_uuid":"366ce68fbc3461163933305ac57c507861bb66e7"},"cell_type":"code","source":"#不用stacking\nres_lgb = []\nfeature_imp = []\nfor tr,va in KFold(n_splits=10,random_state=2018).split(train_feature,train['target'].values):\n    lgbmr = LGBMRegressor(num_leaves=16,n_estimators=100,colsample_bytree=0.7,subsample=0.7,)\n    lgbmr.fit(train_feature[tr],train['target'].values[tr],\n              eval_set=[(train_feature[tr],train['target'].values[tr]),(train_feature[va],train['target'].values[va])],\n              eval_metric='rmse',\n              verbose=50)\n    feature_imp.append(lgbmr.feature_importances_)\n    res_lgb.append(lgbmr.predict(test_feature))\nf = np.mean(feature_imp,axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a9731233e0c7346f2b2d76bf2c974f831783f1b"},"cell_type":"code","source":"avg_res = np.mean(res_lgb,axis=0)\ntest['target'] = avg_res\ntest[['card_id','target']].to_csv('predict1216.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"de298502f922e8cc7064f9a6c476f044abb5ebb0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false},"varInspector":{"cols":{"lenName":16,"lenType":16,"lenVar":40},"kernels_config":{"python":{"delete_cmd_postfix":"","delete_cmd_prefix":"del ","library":"var_list.py","varRefreshCmd":"print(var_dic_list())"},"r":{"delete_cmd_postfix":") ","delete_cmd_prefix":"rm(","library":"var_list.r","varRefreshCmd":"cat(var_dic_list()) "}},"types_to_exclude":["module","function","builtin_function_or_method","instance","_Feature"],"window_display":false}},"nbformat":4,"nbformat_minor":1}