{"cells":[{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T15:57:14.066748Z","start_time":"2018-12-13T15:57:10.939762Z"},"trusted":false,"_uuid":"73605c258ffc83e59374612bb7343826723124ac"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport gc\nimport os\nimport sys\nimport datetime\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm_notebook\nfrom scipy.sparse import hstack,csr_matrix\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\nfrom collections import Counter\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom joblib import Parallel,delayed\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error,mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.feature_selection import SelectPercentile\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import OneHotEncoder,StandardScaler,LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"1c49ee0709432337e03edf0b085507437b30cfc3"},"cell_type":"markdown","source":"## 读入数据"},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T15:59:44.030017Z","start_time":"2018-12-13T15:57:14.467681Z"},"trusted":false,"_uuid":"af673e9f6fe6a198046dd10f73901c97950061d9"},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv',)\ntest = pd.read_csv('../input/test.csv')\nht = pd.read_csv('../input/historical_transactions.csv')\nnmt = pd.read_csv('../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc38f013c6f6d30dd43a789b3f181c2781b31700"},"cell_type":"markdown","source":"## 数据预处理"},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:00:07.009562Z","start_time":"2018-12-13T15:59:44.793663Z"},"collapsed":true,"trusted":false,"_uuid":"f6a8f722a91b538fe82e6f2b5c97d7ef913208d7"},"cell_type":"code","source":"#日期变换\ntrain['first_active_month'] = pd.to_datetime(train['first_active_month'])\ntest['first_active_month'] = pd.to_datetime(test['first_active_month'])\nht['purchase_date'] = pd.to_datetime(ht['purchase_date'])\nht['purchase_year'] = ht['purchase_date'].dt.year\nht['purchase_year'] = ht['purchase_year'].astype(np.int32)\nht['purchase_hour'] = ht['purchase_date'].dt.hour\nht['purchase_month'] = ht['purchase_date'].dt.month\nht['purchase_month'] = ht['purchase_month'].astype(np.int16)\nnmt['purchase_date'] = pd.to_datetime(nmt['purchase_date'])\nnmt['purchase_year'] = nmt['purchase_date'].dt.year\nnmt['purchase_month'] = nmt['purchase_date'].dt.month\nnmt['purchase_hour'] = nmt['purchase_date'].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:00:32.247916Z","start_time":"2018-12-13T16:00:07.566077Z"},"collapsed":true,"trusted":false,"_uuid":"78372d4a40cbfb7260cce08cb392baf2376ce355"},"cell_type":"code","source":"#缺失值填充\nht['category_2'] = ht['category_2'].fillna(0)\nht['category_3'] = ht['category_3'].fillna('E')\nnmt['category_2'] = nmt['category_2'].fillna(0)\nnmt['category_3'] = nmt['category_3'].fillna('E')\nht['category_2'] = ht['category_2'].astype(np.int16)\nnmt['category_2'] = nmt['category_2'].astype(np.int16)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"7098e166662627a36527e64908bad27555637510"},"cell_type":"code","source":"train['feature_1'] = train['feature_1'].astype(str)\ntrain['feature_2'] = train['feature_2'].astype(str)\ntrain['feature_3'] = train['feature_3'].astype(str)\ntest['feature_1'] = test['feature_1'].astype(str)\ntest['feature_2'] = test['feature_2'].astype(str)\ntest['feature_3'] = test['feature_3'].astype(str)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a505ddcd5f7a8b76df80cf9e4dbcdd845c16e87c"},"cell_type":"markdown","source":"## 特征工程"},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:01:50.699145Z","start_time":"2018-12-13T16:01:40.636998Z"},"collapsed":true,"trusted":false,"_uuid":"fbd5aed685ca5ba68ab6cba1bb2a9370ed23850b"},"cell_type":"code","source":"#激活日期距离今天的时间\ntrain['active_till_now'] = train['first_active_month'].map(lambda x:(datetime.datetime.today() - x).days)\ntest['active_till_now'] = test['first_active_month'].map(lambda x:(datetime.datetime.today() - x).days)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"afded0fb9ce34fd59d2bd9a9f3df8950a22a5d5c"},"cell_type":"markdown","source":"### 针对历史记录进行特征构造"},{"metadata":{"trusted":false,"_uuid":"fe5a545c18097f90716e8e2453425cebf9369193"},"cell_type":"code","source":"#统计在24小时内的购买记录次数\nhour_purchase_cnt = pd.pivot_table(ht,values='month_lag',index='card_id',columns=['purchase_hour'],aggfunc=len,fill_value=0)\nhour_purchase_cnt.columns = ['purchase_hour_%s'%str(i) for i in range(24)]\nhour_purchase_cnt.reset_index(inplace=True)\ntrain = pd.merge(train,hour_purchase_cnt,on='card_id',how='left')\ntest = pd.merge(test,hour_purchase_cnt,on='card_id',how='left')\ndel hour_purchase_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f7d596ac9cc7850d4b934c760016f0ded26d91eb"},"cell_type":"code","source":"#统计在24小时内的购买记录次数\nhour_sub_id = pd.pivot_table(ht,values='month_lag',index='card_id',columns=['subsector_id'],aggfunc=len,fill_value=0)\nhour_sub_id.columns = ['sub_id_%s'%str(i) for i in range(ht['subsector_id'].nunique())]\nhour_sub_id.reset_index(inplace=True)\ntrain = pd.merge(train,hour_sub_id,on='card_id',how='left')\ntest = pd.merge(test,hour_sub_id,on='card_id',how='left')\ndel hour_sub_id\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a7e1d5c070cc303681940f8148803a86fd18f110"},"cell_type":"code","source":"#统计在历史上每个月的购买次数\nym_purchase_cnt = pd.pivot_table(ht,values='month_lag',index='card_id',columns=['purchase_year','purchase_month'],aggfunc=len,fill_value=0)\nym_purchase_cnt.columns = ['purchase_ym_%s'%str(i) for i in range(ym_purchase_cnt.shape[1])]\nym_purchase_cnt.reset_index(inplace=True)\ntrain = pd.merge(train,ym_purchase_cnt,on='card_id',how='left')\ntest = pd.merge(test,ym_purchase_cnt,on='card_id',how='left')\ndel ym_purchase_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"925e483d5e21588dcafdb89d20e52cbd5c18f3df"},"cell_type":"code","source":"#统计在24小时内的平均购买金额\nhour_purchase_amount = pd.pivot_table(ht,values='purchase_amount',index='card_id',columns=['purchase_hour'],aggfunc=np.mean,fill_value=0)\nhour_purchase_amount.columns = ['purchase_amount_%s'%str(i) for i in range(24)]\nhour_purchase_amount.reset_index(inplace=True)\ntrain = pd.merge(train,hour_purchase_amount,on='card_id',how='left')\ntest = pd.merge(test,hour_purchase_amount,on='card_id',how='left')\ndel hour_purchase_amount\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"d1bc884766192fa2a872b80d18d397c5e0d6c0f8"},"cell_type":"code","source":"#统计在24小时内的平均month_lag\nhour_month_lag = pd.pivot_table(ht,values='month_lag',index='card_id',columns=['purchase_hour'],aggfunc=np.sum,fill_value=0)\nhour_month_lag.columns = ['month_lag_%s'%str(i) for i in range(24)]\nhour_month_lag.reset_index(inplace=True)\ntrain = pd.merge(train,hour_month_lag,on='card_id',how='left')\ntest = pd.merge(test,hour_month_lag,on='card_id',how='left')\ndel hour_month_lag\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b2ef62b4a1fdc25f6f78ed3975a30a5381673805"},"cell_type":"code","source":"#统计在24小时内的平均month_lag\nhour_installment = pd.pivot_table(ht,values='installments',index='card_id',columns=['purchase_hour'],aggfunc=np.sum,fill_value=0)\nhour_installment.columns = ['month_installments_%s'%str(i) for i in range(24)]\nhour_installment.reset_index(inplace=True)\ntrain = pd.merge(train,hour_installment,on='card_id',how='left')\ntest = pd.merge(test,hour_installment,on='card_id',how='left')\ndel hour_installment\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:02:00.100704Z","start_time":"2018-12-13T16:01:51.063176Z"},"trusted":false,"_uuid":"53b9fd8687017606443e676bba77edc83e06ec43"},"cell_type":"code","source":"#统计历史上总共买过多少次东西\nht_cnt = ht.groupby('card_id')['city_id'].count().reset_index()\nht_cnt.columns = ['card_id','ht_cnt']\n#merge the result\ntrain = pd.merge(train,ht_cnt,on='card_id',how='left')\ntest = pd.merge(test,ht_cnt,on='card_id',how='left')\ndel ht_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7b8b3318877b41286cb872d2bedce60e6052136b"},"cell_type":"code","source":"#计算历史上的消费状况\nht_amount = ht.groupby('card_id').agg({'purchase_amount':[np.max,np.mean,np.std]})\nht_amount.columns = ['amount_max','amount_mean','amount_std']\nht_amount = ht_amount.reset_index()\ntrain = pd.merge(train,ht_amount,on='card_id',how='left')\ntest = pd.merge(test,ht_amount,on='card_id',how='left')\ndel ht_amount\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e94a73f6cd8e401288b9f30478ee4d0bda608d97"},"cell_type":"code","source":"#计算历史上month_lag的情况\nht_month_lag = ht.groupby('card_id').agg({'month_lag':[np.min,np.mean,np.std]})\nht_month_lag.columns = ['lag_min','lag_mean','lag_std']\nht_month_lag = ht_month_lag.reset_index()\ntrain = pd.merge(train,ht_month_lag,on='card_id',how='left')\ntest = pd.merge(test,ht_month_lag,on='card_id',how='left')\ndel ht_month_lag\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:03:03.498767Z","start_time":"2018-12-13T16:02:40.618914Z"},"trusted":false,"_uuid":"13627eb2b0521797dc6d6eeddb3b28ef08fb7601"},"cell_type":"code","source":"#计算历史上installments的情况\nht_installment = ht.groupby('card_id').agg({'installments':[np.max,np.min,np.mean,np.std]})\nht_installment.columns = ['ins_max','ins_min','ins_mean','ins_std']\nht_installment = ht_installment.reset_index()\ntrain = pd.merge(train,ht_installment,on='card_id',how='left')\ntest = pd.merge(test,ht_installment,on='card_id',how='left')\ndel ht_installment\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e9ca0c5529ce6f90ef39fc9d396c2c04de260ce4"},"cell_type":"code","source":"ht_auth_flag = pd.pivot_table(ht,values='city_id',index='card_id',columns='authorized_flag',aggfunc=len,fill_value=0)\nht_auth_flag.columns = ['auth_y','auth_n']\nht_auth_flag = ht_auth_flag.reset_index()\ntrain = pd.merge(train,ht_auth_flag,on='card_id',how='left')\ntest = pd.merge(test,ht_auth_flag,on='card_id',how='left')\ndel ht_auth_flag\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cfff936aa78d2201a55c1df76559a1002a69ac34"},"cell_type":"code","source":"ht['merchant_category_id'] = ht['merchant_category_id'].map(lambda x:str(x))\nht_mer_cate = ht.groupby('card_id').apply(lambda x:' '.join(x['merchant_category_id']))\nht_mer_cate = ht_mer_cate.reset_index()\nht_mer_cate.columns = ['card_id','mer_cate_id_list']\ntrain = pd.merge(train,ht_mer_cate,on='card_id',how='left')\ntest = pd.merge(test,ht_mer_cate,on='card_id',how='left')\ndel ht_mer_cate\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"d66720408b4256829fb1e155acf035ff032c44a4"},"cell_type":"code","source":"ht_state_cnt = ht.groupby('card_id')['state_id'].nunique().reset_index()\nht_state_cnt.columns = ['card_id','state_cnt']\ntrain = pd.merge(train,ht_state_cnt,on='card_id',how='left')\ntest = pd.merge(test,ht_state_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9e377809639d3a9cae2579dca98af8e284a01f6f"},"cell_type":"code","source":"del ht_state_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"642cb339cd31bdd4b10331050f72b3f513ad8b0a"},"cell_type":"code","source":"ht_mer_cate_cnt = ht.groupby('card_id')['merchant_category_id'].nunique().reset_index()\nht_mer_cate_cnt.columns = ['card_id','merchant_category_id_cnt']\ntrain = pd.merge(train,ht_mer_cate_cnt,on='card_id',how='left')\ntest = pd.merge(test,ht_mer_cate_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"42d73511af8fce51dc9ae373ceb0d96643721dfa"},"cell_type":"code","source":"del ht_mer_cate_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"81d2627a95bf6dbd53023ebb38de9bd8ef7a4232"},"cell_type":"code","source":"ht_mer_cnt = ht.groupby('card_id')['merchant_id'].nunique().reset_index()\nht_mer_cnt.columns = ['card_id','merchant_id_cnt']\ntrain = pd.merge(train,ht_mer_cnt,on='card_id',how='left')\ntest = pd.merge(test,ht_mer_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7c7b21a0dbdcb6a64850e5a8ed64a6576d1e2c5e"},"cell_type":"code","source":"del ht_mer_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ed3b8114fa8c3179247cab5e948379195ca1d91"},"cell_type":"markdown","source":"### 针对新商店记录进行特征构造"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"aabce1243a844135bac002aab77bcbc9d9a3a6b4"},"cell_type":"code","source":"#统计在新商店的24小时的购买记录次数分布\nnew_hour_purchase_cnt = pd.pivot_table(nmt,values='month_lag',index='card_id',columns=['purchase_hour'],aggfunc=len,fill_value=0)\nnew_hour_purchase_cnt.columns = ['new_purchase_hour_%s'%str(i) for i in range(24)]\nnew_hour_purchase_cnt.reset_index(inplace=True)\ntrain = pd.merge(train,new_hour_purchase_cnt,on='card_id',how='left')\ntest = pd.merge(test,new_hour_purchase_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d2c38be4c676361fbb707098686125ec94627e99"},"cell_type":"code","source":"del new_hour_purchase_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4c983cfcd58c3254ccd6438ae650e5a563950f11"},"cell_type":"code","source":"#统计在24小时内的购买记录次数\nnew_hour_sub_id = pd.pivot_table(nmt,values='month_lag',index='card_id',columns=['subsector_id'],aggfunc=len,fill_value=0)\nnew_hour_sub_id.columns = ['new_sub_id_%s'%str(i) for i in range(nmt['subsector_id'].nunique())]\nnew_hour_sub_id.reset_index(inplace=True)\ntrain = pd.merge(train,new_hour_sub_id,on='card_id',how='left')\ntest = pd.merge(test,new_hour_sub_id,on='card_id',how='left')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fa9a3842904759f53f2033f621424b336bc2a19b"},"cell_type":"code","source":"del new_hour_sub_id\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"87d6142087bb1711beaac81f9416c14c738aebd8"},"cell_type":"code","source":"#统计在历史上每个月的购买次数\nnew_ym_purchase_cnt = pd.pivot_table(nmt,values='month_lag',index='card_id',columns=['purchase_year','purchase_month'],aggfunc=len,fill_value=0)\nnew_ym_purchase_cnt.columns = ['new_purchase_ym_%s'%str(i) for i in range(new_ym_purchase_cnt.shape[1])]\nnew_ym_purchase_cnt.reset_index(inplace=True)\ntrain = pd.merge(train,new_ym_purchase_cnt,on='card_id',how='left')\ntest = pd.merge(test,new_ym_purchase_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"70b24dd9d8d009447af4663bb7fce88f74853f92"},"cell_type":"code","source":"del new_ym_purchase_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"9db1aeaea3d4a2c865fb18ddcc2699870efcdd68"},"cell_type":"code","source":"#统计在24小时内的平均购买金额\nnmt_hour_purchase_amount = pd.pivot_table(nmt,values='purchase_amount',index='card_id',columns=['purchase_hour'],aggfunc=np.mean,fill_value=0)\nnmt_hour_purchase_amount.columns = ['new_purchase_amount_%s'%str(i) for i in range(24)]\nnmt_hour_purchase_amount.reset_index(inplace=True)\ntrain = pd.merge(train,nmt_hour_purchase_amount,on='card_id',how='left')\ntest = pd.merge(test,nmt_hour_purchase_amount,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a7bbe1e2a46360761de3ea24801c9081d06d548b"},"cell_type":"code","source":"del nmt_hour_purchase_amount\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"62ad8875594d97c6381a42b4c2a44df3072a3736"},"cell_type":"code","source":"#统计在24小时内的平均购买金额\nnmt_hour_month_lag = pd.pivot_table(nmt,values='month_lag',index='card_id',columns=['purchase_hour'],aggfunc=np.mean,fill_value=0)\nnmt_hour_month_lag.columns = ['new_month_lag_%s'%str(i) for i in range(24)]\nnmt_hour_month_lag.reset_index(inplace=True)\ntrain = pd.merge(train,nmt_hour_month_lag,on='card_id',how='left')\ntest = pd.merge(test,nmt_hour_month_lag,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3cb94cfb5bf46751711e57d0748045e53ee4ab70"},"cell_type":"code","source":"del nmt_hour_month_lag\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"dfe4557f4b3a4cfc23b3aa55db10e30f412c01a3"},"cell_type":"code","source":"#统计在24小时内的平均购买金额\nnmt_hour_installments = pd.pivot_table(nmt,values='installments',index='card_id',columns=['purchase_hour'],aggfunc=np.mean,fill_value=0)\nnmt_hour_installments.columns = ['new_installments_%s'%str(i) for i in range(24)]\nnmt_hour_installments.reset_index(inplace=True)\ntrain = pd.merge(train,nmt_hour_installments,on='card_id',how='left')\ntest = pd.merge(test,nmt_hour_installments,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"28154add7720e5e0d21baea4dcc758257dd14d44"},"cell_type":"code","source":"del nmt_hour_installments\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"14ac1f8d96d51a655194b260aff8261bdece4b6f"},"cell_type":"code","source":"#统计历史上总共买过多少次东西\nnmt_cnt = nmt.groupby('card_id')['city_id'].count().reset_index()\nnmt_cnt.columns = ['card_id','nmt_cnt']\n#merge the result\ntrain = pd.merge(train,nmt_cnt,on='card_id',how='left')\ntest = pd.merge(test,nmt_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1265ff607dc6448255fb5c6ceee1e4a520d211fc"},"cell_type":"code","source":"del nmt_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"bb3a77dfe52ad13ab317191bf46f729449a65bfd"},"cell_type":"code","source":"#计算历史上的消费状况\nnmt_amount = nmt.groupby('card_id').agg({'purchase_amount':[np.max,np.mean,np.std]})\nnmt_amount.columns = ['nmt_amount_max','nmt_amount_mean','nmt_amount_std']\nnmt_amount = nmt_amount.reset_index()\ntrain = pd.merge(train,nmt_amount,on='card_id',how='left')\ntest = pd.merge(test,nmt_amount,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"36f16937cb64c5a31b943fbabc00b2a89f354742"},"cell_type":"code","source":"del nmt_amount\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f19c388a635d54008c434e143233db1ba564f904"},"cell_type":"code","source":"#计算历史上month_lag的情况\nnmt_month_lag = nmt.groupby('card_id').agg({'month_lag':[np.min,np.mean,np.std]})\nnmt_month_lag.columns = ['nmt_lag_min','nmt_lag_mean','nmt_lag_std']\nnmt_month_lag = nmt_month_lag.reset_index()\ntrain = pd.merge(train,nmt_month_lag,on='card_id',how='left')\ntest = pd.merge(test,nmt_month_lag,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e37a30f2571181464604a465463d5a606345df79"},"cell_type":"code","source":"del nmt_month_lag\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"bda12db3d8533285b1766da4cde996382cd0fc66"},"cell_type":"code","source":"#计算历史上installments的情况\nnmt_installment = nmt.groupby('card_id').agg({'installments':[np.max,np.min,np.mean,np.std]})\nnmt_installment.columns = ['nmt_ins_max','nmt_ins_min','nmt_ins_mean','nmt_ins_std']\nnmt_installment = nmt_installment.reset_index()\ntrain = pd.merge(train,nmt_installment,on='card_id',how='left')\ntest = pd.merge(test,nmt_installment,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8104514f661ea5e36155ee89f22813d349a09c23"},"cell_type":"code","source":"del nmt_installment\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"56f9c64694446f386408711f9c180237b1716800"},"cell_type":"code","source":"nmt_auth_flag = pd.pivot_table(ht,values='city_id',index='card_id',columns='authorized_flag',aggfunc=len,fill_value=0)\nnmt_auth_flag.columns = ['nmt_auth_y','nmt_auth_n']\nnmt_auth_flag = nmt_auth_flag.reset_index()\ntrain = pd.merge(train,nmt_auth_flag,on='card_id',how='left')\ntest = pd.merge(test,nmt_auth_flag,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"258a8f8012f50cbe27ea24b3c7a6c73f55fdc981"},"cell_type":"code","source":"del nmt_auth_flag\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"7271b6a236486399a0065b2fe908f7346115fd90"},"cell_type":"code","source":"nmt['merchant_category_id'] = ht['merchant_category_id'].map(lambda x:str(x))\nnmt_mer_cate = nmt.groupby('card_id').apply(lambda x:' '.join(x['merchant_category_id']))\nnmt_mer_cate = nmt_mer_cate.reset_index()\nnmt_mer_cate.columns = ['card_id','nmt_mer_cate_id_list']\ntrain = pd.merge(train,nmt_mer_cate,on='card_id',how='left')\ntest = pd.merge(test,nmt_mer_cate,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"09064b65d3ea3d77a0f408b732cab9e4cd946795"},"cell_type":"code","source":"train['nmt_mer_cate_id_list'] = train['nmt_mer_cate_id_list'].fillna('')\ntest['nmt_mer_cate_id_list'] = test['nmt_mer_cate_id_list'].fillna('')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4c2f10285d97bf0309474790abdac12f9e68cde2"},"cell_type":"code","source":"del nmt_mer_cate\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"dcbf5eafab9b7ef124a8da41385e2773e062c1a2"},"cell_type":"code","source":"nmt_state_cnt = nmt.groupby('card_id')['state_id'].nunique().reset_index()\nnmt_state_cnt.columns = ['card_id','nmt_state_cnt']\ntrain = pd.merge(train,nmt_state_cnt,on='card_id',how='left')\ntest = pd.merge(test,nmt_state_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8b38baa53e64433a22520bbd99b46bd9cd64a1e7"},"cell_type":"code","source":"del nmt_state_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a7e7a9d1a88a17d81dd67750bc925171c7ce24d2"},"cell_type":"code","source":"nmt_mer_cate_cnt = nmt.groupby('card_id')['merchant_category_id'].nunique().reset_index()\nnmt_mer_cate_cnt.columns = ['card_id','nmt_merchant_category_id_cnt']\ntrain = pd.merge(train,nmt_mer_cate_cnt,on='card_id',how='left')\ntest = pd.merge(test,nmt_mer_cate_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0a5b6c5fcc8995e6a9d88822347aacaf11b3947b"},"cell_type":"code","source":"del nmt_mer_cate_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"644d9ca515490d8df03437dce55b88fe6b1f9df3"},"cell_type":"code","source":"nmt_mer_cnt = nmt.groupby('card_id')['merchant_id'].nunique().reset_index()\nnmt_mer_cnt.columns = ['card_id','nmt_merchant_id_cnt']\ntrain = pd.merge(train,nmt_mer_cnt,on='card_id',how='left')\ntest = pd.merge(test,nmt_mer_cnt,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"61f209ff125ee2ab60df0ce0c0961a8af3e585f3"},"cell_type":"code","source":"del nmt_mer_cnt\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ec20d9c84bc85a4e4a6dbd38cf2e3411c2615140"},"cell_type":"code","source":"train_cate = pd.get_dummies(train[['feature_1','feature_2','feature_3']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"25db9a1fb9370748a824b6628b347c75c9e914e7"},"cell_type":"code","source":"test_cate = pd.get_dummies(test[['feature_1','feature_2','feature_3']])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca3acb1da37b5eb19c55a68ce14b471eabeb5e32"},"cell_type":"markdown","source":"### 整理稀疏特征"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"95a16a9de5994b395b6b9c04229aac1e66ec1a71"},"cell_type":"code","source":"cv = CountVectorizer()\ncv_train = cv.fit_transform(train['mer_cate_id_list'])\ncv_test =  cv.transform(test['mer_cate_id_list'])\ntrain.drop('mer_cate_id_list',axis=1,inplace=True)\ntest.drop('mer_cate_id_list',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"97b2282d5e59ec06b0937eaa6fa246bb2e18638e"},"cell_type":"code","source":"ss = StandardScaler(with_mean=False)\ncv_train = ss.fit_transform(cv_train)\ncv_test = ss.transform(cv_test)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c29b01b743150aff94e9d8a2fd4532d789ccfdc2"},"cell_type":"code","source":"cv_ = CountVectorizer()\ncv_train_ = cv_.fit_transform(train['nmt_mer_cate_id_list'])\ncv_test_ =  cv_.transform(test['nmt_mer_cate_id_list'])\ntrain.drop('nmt_mer_cate_id_list',axis=1,inplace=True)\ntest.drop('nmt_mer_cate_id_list',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"230f6d45859c0678ea1606791c3ef09cf21e9a4e"},"cell_type":"code","source":"ss = StandardScaler(with_mean=False)\ncv_train_ = ss.fit_transform(cv_train_)\ncv_test_ = ss.transform(cv_test_)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"c16d4e2a76d421acfc3e0b01a5ea18cbffed8a57"},"cell_type":"markdown","source":"### 特征归一化和拼接"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"940e577baa857457a8cbde5bf21d0f1742e978c8"},"cell_type":"code","source":"train.fillna(0,inplace=True)\ntest.fillna(0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:05.311601Z","start_time":"2018-12-13T16:07:05.304619Z"},"collapsed":true,"trusted":false,"_uuid":"280c0d2464f23461d3271e0f4342f60e08c4c2a9"},"cell_type":"code","source":"#handler the numeric feature\ndrop_columns = ['first_active_month','card_id','feature_1','feature_2','feature_3','target','year','month','cate']\nnum_feature = [v for v in train.columns if v not in drop_columns]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:08.507123Z","start_time":"2018-12-13T16:07:07.438186Z"},"collapsed":true,"trusted":false,"_uuid":"a9d9026f37c4f10300a8256ff9c776c1782fbc78"},"cell_type":"code","source":"train_num_feature = train[num_feature].values\ntest_num_feature = test[num_feature].values\nss = StandardScaler()\ntrain_num_feature = ss.fit_transform(train_num_feature)\ntest_num_feature = ss.transform(test_num_feature)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-12-13T16:07:16.726010Z","start_time":"2018-12-13T16:07:16.083803Z"},"trusted":false,"_uuid":"2abf43127f2a4a3d1f50cf2603de2adb2d8d96fc"},"cell_type":"code","source":"#stack the feature\ntrain_feature = hstack([train_num_feature,cv_train,cv_train_,train_cate]).tocsr()\ntest_feature = hstack([test_num_feature,cv_test,cv_test_,test_cate]).tocsr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e446ddeefd9ae60fd89d0a9ad2391d941f17a745"},"cell_type":"code","source":"del train_num_feature\ndel test_num_feature\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ed44c1277795dbc580f90888074253443d9270f"},"cell_type":"markdown","source":"## 训练模型和预测数据"},{"metadata":{"trusted":false,"_uuid":"f12f233a1717f83ae19d00d0e379dadd7521c3b0"},"cell_type":"code","source":"#不用stacking\nres_lgb = []\nfeature_imp = []\nfor tr,va in KFold(n_splits=10,random_state=2018).split(train_feature,train['target'].values):\n    lgbmr = LGBMRegressor(num_leaves=32,n_estimators=150,colsample_bytree=0.7,subsample=0.7)\n    lgbmr.fit(train_feature[tr],train['target'].values[tr],early_stopping_rounds=10,\n              eval_set=[(train_feature[tr],train['target'].values[tr]),(train_feature[va],train['target'].values[va])],\n              eval_metric='rmse',\n              verbose=50)\n    feature_imp.append(lgbmr.feature_importances_)\n    res_lgb.append(lgbmr.predict(test_feature))\nf = np.mean(feature_imp,axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"95c3a85c976ff9b86d977daadd66cf44846fa443"},"cell_type":"code","source":"avg_res = np.mean(res_lgb,axis=0)\ntest['target'] = avg_res\ntest[['card_id','target']].to_csv('predictlgb_1225.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"74f09e6c44f108edbc11cabf46bd3a6374c3497b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.2"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{"height":"calc(100% - 180px)","left":"10px","top":"150px","width":"255px"},"toc_section_display":true,"toc_window_display":false},"varInspector":{"cols":{"lenName":16,"lenType":16,"lenVar":40},"kernels_config":{"python":{"delete_cmd_postfix":"","delete_cmd_prefix":"del ","library":"var_list.py","varRefreshCmd":"print(var_dic_list())"},"r":{"delete_cmd_postfix":") ","delete_cmd_prefix":"rm(","library":"var_list.r","varRefreshCmd":"cat(var_dic_list()) "}},"types_to_exclude":["module","function","builtin_function_or_method","instance","_Feature"],"window_display":false}},"nbformat":4,"nbformat_minor":1}