{"cells":[{"metadata":{"_uuid":"bd6912b0e9c5d09def2f485329cc5332f0b2d593"},"cell_type":"markdown","source":"# Feature Engineering + LightGBM Model with Python\n"},{"metadata":{"_uuid":"6dd24ff1fa7479bde7e4b8314e04f02cccd4fb4b"},"cell_type":"markdown","source":"### My another Kernel with R : Feature Engineering + XGBoost + LGBM with R\n\n(It's my first Kernel, so it might be a little bit messy -.-)\n\nhttps://www.kaggle.com/juliaflower/feature-engineering-xgboost-lgbm-with-r"},{"metadata":{"_uuid":"38e86350d9f5060ef5e2fd62673da0a4bd5eb9e0"},"cell_type":"markdown","source":"Reference\n\nhttps://www.kaggle.com/fabiendaniel/elo-world\n\nhttps://www.kaggle.com/ashishpatel26/repeated-kfold-approach-rmse-3-70\n\nhttps://www.kaggle.com/chauhuynh/my-first-kernel-3-699/\n\nhttps://www.kaggle.com/yhn112/data-exploration-lightgbm-catboost-lb-3-760\n\nhttps://www.kaggle.com/nikitsoftweb/you-re-going-to-want-more-categories-lb-3-70"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import RepeatedKFold\nfrom sklearn.metrics import mean_squared_error\nimport datetime\nimport time\nimport sys\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b2dcee2d59f223b518a07a05a8f51fb70f6752f","trusted":true},"cell_type":"code","source":"# loading original data \ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nhistdata = pd.read_csv(\"../input/historical_transactions.csv\")\nnewdata = pd.read_csv(\"../input/new_merchant_transactions.csv\")\nmerchants = pd.read_csv(\"../input/merchants.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d37c4dfc4acafcede474209206bd3907cf038f0"},"cell_type":"code","source":"# for submition\nlgb_submition = pd.DataFrame({\"card_id\":test[\"card_id\"].values})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a6c2ecc0b00c88df554cc0706e720071c59b6eb"},"cell_type":"markdown","source":"## Feature Engineering"},{"metadata":{"trusted":true,"_uuid":"35770d76122e6914bd7fde649489d3d6e1fa4b9f"},"cell_type":"code","source":"# train & test data\n# extract target value\ntarget = train['target']\n\n# Convert time as features\nfor data in [train,test]:\n    data['first_active_month'] = pd.to_datetime(data['first_active_month'])\n    data['year'] = data['first_active_month'].dt.year\n    data['month'] = data['first_active_month'].dt.month\n    data['howlong'] = (datetime.date(2018,2,1) - data['first_active_month'].dt.date).dt.days\n\ntrain = train.drop(['first_active_month','target'], axis=1)\ntest = test.drop(['first_active_month'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d38da3f0af26aec3bd43b0bd037ba4bfbb8db8d"},"cell_type":"code","source":"# Convert category values\ndef category_convert(data):\n    data['cat2'] = data['category_2']\n    data['cat3'] = data['category_3']\n    data = pd.get_dummies(data, columns=['cat2', 'cat3'])\n    for bi_cat in ['authorized_flag', 'category_1']:\n        data[bi_cat] = data[bi_cat].map({'Y':1, 'N':0})\n    return data\n\nhistdata = category_convert(histdata)\nnewdata = category_convert(newdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"605b6d8f6b3283dfd1d5ce4cd6b72111663c5647"},"cell_type":"code","source":"# historical_transactions & new merchants transaction\n# categorical data and other general data\ndef aggregate_trans(data, prefix):  \n    agg_func = {\n        'card_id': ['size'], #num_trans\n        'authorized_flag': ['sum', 'mean','nunique'],\n        'category_1': ['sum', 'mean','nunique'],\n        'category_2': ['nunique'],\n        'category_3': ['nunique'],\n        'cat2_1.0': ['mean'],\n        'cat2_2.0': ['mean'],\n        'cat2_3.0': ['mean'],\n        'cat2_4.0': ['mean'],\n        'cat2_5.0': ['mean'],\n        'cat3_A': ['mean'],\n        'cat3_B': ['mean'],\n        'cat3_C': ['mean'],\n        'city_id': ['nunique'],\n        'state_id': ['nunique'],\n        'subsector_id': ['nunique'],\n        'installments': ['sum', 'mean','median', 'max', 'min', 'std', 'nunique'],\n        'merchant_id': ['nunique'],\n        'merchant_category_id': ['nunique'],\n        'month_lag': ['mean', 'max', 'min', 'std', 'nunique'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std', 'nunique']\n    }    \n    agg_trans = data.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    return agg_trans\n\nhist_sum = aggregate_trans(histdata, 'hist_')\nnew_sum = aggregate_trans(newdata, 'new_')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"faf6b762cf99959a51066d1480b99c2045d525da"},"cell_type":"code","source":"# Devide time \n# Code from: Chau Ngoc Huynh - \"My first kernel (3.699)\"\n\ndef devide_time(data):\n    data['purchase_date'] = pd.to_datetime(data['purchase_date'])\n    data['month_diff'] = ((datetime.datetime.today() - data['purchase_date']).dt.days)//30  \n    data['purchase_year'] = data['purchase_date'].dt.year\n    data['purchase_month'] = data['purchase_date'].dt.month\n    data['weekofyear'] = data['purchase_date'].dt.weekofyear\n    data['dayofweek'] = data['purchase_date'].dt.dayofweek\n    data['weekend'] = (data.purchase_date.dt.weekday >=5).astype(int)\n    data['hour'] = data['purchase_date'].dt.hour\n    return data\n\nhist_times = devide_time(histdata)\nnew_times = devide_time(newdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0ebdc5290e2b327b96f8b799c1160d98004f640"},"cell_type":"code","source":"def aggregate_times(data, prefix):  \n#     data.loc[:, 'purchase_date'] = pd.DatetimeIndex(data['purchase_date']).astype(np.int64) * 1e-9\n\n    agg_func = {\n#         'purchase_date': [np.ptp, 'min', 'max','nunique'],  #np.ptp=pur_term\n        'month_diff': ['mean','max','min'],\n        'purchase_year': ['mean', 'max', 'min', 'std','nunique'],\n        'purchase_month': ['mean', 'max', 'min', 'std','nunique'],\n        'weekofyear': ['mean','max','min','nunique'],\n        'dayofweek': ['mean'],\n        'weekend': ['sum', 'mean'],\n        'hour': ['mean','max','min']\n    }    \n    agg_times = data.groupby(['card_id']).agg(agg_func)\n    agg_times.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_times.columns.values]\n    agg_times.reset_index(inplace=True)\n    \n    return agg_times\n\nhist_times = aggregate_times(hist_times, 'hist_')\nnew_times = aggregate_times(new_times, 'new_')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b551fecf8646c514fb02a2ae00aeba413aba2fb"},"cell_type":"code","source":"# purchase date term\nhistdata['pur_date'] = pd.DatetimeIndex(histdata['purchase_date']).date\nnewdata['pur_date'] = pd.DatetimeIndex(newdata['purchase_date']).date\n\nhistdata.loc[:,'pur_date'] = pd.DatetimeIndex(histdata['pur_date']).astype(np.int64) * 1e-9\nnewdata.loc[:,'pur_date'] = pd.DatetimeIndex(newdata['pur_date']).astype(np.int64) * 1e-9\n\nagg_fn= {\n        'pur_date': [np.ptp,'max','min'], # np.ptp: Range of values (maximum - minimum) \n        }\nagg_hist = histdata.groupby(['card_id']).agg(agg_fn)\nagg_hist.columns = ['_'.join(col).strip() for col in agg_hist.columns.values]\nagg_hist.reset_index(inplace=True)\n\nagg_new = newdata.groupby(['card_id']).agg(agg_fn)\nagg_new.columns = ['_'.join(col).strip() for col in agg_new.columns.values]\nagg_new.reset_index(inplace=True)\n\nagg_hist.columns = ['hist_' + c if c != 'card_id' else c for c in agg_hist.columns]\nagg_new.columns = ['new_' + c if c != 'card_id' else c for c in agg_new.columns]\n\n# scale agg_hist, agg_new\nimport sklearn as sk\nfrom sklearn import preprocessing\n\nagg_hist['hist_pur_date_ptp']=sk.preprocessing.scale(agg_hist['hist_pur_date_ptp'])\nagg_new['new_pur_date_ptp']=sk.preprocessing.scale(agg_new['new_pur_date_ptp'])\nagg_hist['hist_pur_date_max']=sk.preprocessing.scale(agg_hist['hist_pur_date_max'])\nagg_new['new_pur_date_max']=sk.preprocessing.scale(agg_new['new_pur_date_max'])\nagg_hist['hist_pur_date_min']=sk.preprocessing.scale(agg_hist['hist_pur_date_min'])\nagg_new['new_pur_date_min']=sk.preprocessing.scale(agg_new['new_pur_date_min'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6fb67afab3f96768404e788c10f4c909dd0be24"},"cell_type":"code","source":"# merge \nhist = hist_times.merge(hist_sum,on='card_id',how='left')\nhist = hist.merge(agg_hist, on='card_id',how='left')\ndel hist_sum\ndel hist_times\ndel agg_hist\n\nnew = new_times.merge(new_sum, on='card_id',how='left')\nnew = new.merge(agg_new, on='card_id',how='left')\ndel new_sum\ndel new_times\ndel agg_new","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a422042ae3ee1f1b9654ea6a8f56bd58eba11273"},"cell_type":"code","source":"# # avg_term = as.integer(mean(abs(diff(order(purchase_date)))))\n# def avg_term_f(x):\n#     s = x.sort_values()\n#     y = abs(np.diff(s)).mean().tolist()\n#     return y\n\n# hist['hist_avg_term'] = histdata.groupby('card_id')['purchase_date'].apply(avg_term_f)\n# new['new_avg_term'] = newdata.groupby('card_id')['purchase_date'].apply(avg_term_f)\n# still working on....","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48679769e8643f225f4c3a3ac4843cc164cd4b9d"},"cell_type":"code","source":"train = train.merge(hist, on='card_id',how='left')\ntrain = train.merge(new, on='card_id',how='left')\n\ntest = test.merge(hist, on='card_id',how='left')\ntest = test.merge(new, on='card_id',how='left')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"479f958a95147117b9b64c7ec49b612dc485a3f7"},"cell_type":"code","source":"# save featured data\ntrain.to_csv(\"train_featured.csv\", index=False)\ntest.to_csv(\"test_featured.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"baf90f504b1529bd3921f62e4a32c3761514068d"},"cell_type":"code","source":"# drop card_id before running model\ntrain = train.drop('card_id', axis=1) #,'hist_avg_term','new_avg_term'\ntest = test.drop('card_id', axis=1) #,'hist_avg_term','new_avg_term'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed159ef8dc8f78c0786b492123a27a789dcd0b48"},"cell_type":"markdown","source":"## LightGBM Model\nApproach:\n\n1st round training: find out important features -> delete correlated features\n\n2nd round training: final prediction only with selected features"},{"metadata":{"_uuid":"75294563c582639d56ade9e901e556ed39fe7b66"},"cell_type":"markdown","source":"### 1st round: run the model for extracting important features"},{"metadata":{"trusted":true,"_uuid":"94d97903dec5be0b0f8cd572bfe44198bc1f778a"},"cell_type":"code","source":"# set default parameters for 1st round training\nparams = {'boosting': 'gbdt',\n          'objective':'regression',\n          'metric': 'rmse',\n          'learning_rate': 0.01, # 0.003! #0.005 #0.006 \n          'num_leaves': 110, #110 #100 #150 large, but over-fitting\n          'max_bin': 66,  #60 #50 # large,but slower,over-fitting\n          'max_depth': 10, # deal with over-fitting\n          'min_data_in_leaf': 30, # deal with over-fitting\n          'min_child_samples': 20,\n          'feature_fraction': 0.5,#0.5 #0.6 #0.8\n          'bagging_fraction': 0.8,\n          'bagging_freq': 40,#5  \n          'bagging_seed': 11,\n          'lambda_l1': 2,#1.3! #5 #1.2 #1\n          'lambda_l2': 0.1 #0.1\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91b0c81344c8c52f0c5111236719ae1e4fd793d8"},"cell_type":"code","source":"# Reference: code from Ashish Patel(阿希什)Repeated KFOLD Approach: RMSE[3.70]\n# Kfold cross-validation\n# folds = KFold(n_splits=5, shuffle=True, random_state=11)\n\nnfolds = 5\nnrepeats = 2 \nfolds = RepeatedKFold(n_splits=nfolds, n_repeats=nrepeats, random_state=11)\nfold_pred = np.zeros(len(train))\nfeature_importance_df = pd.DataFrame()\nlgb_preds = np.zeros(len(test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values,target.values)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx]) #categorical_feature=categorical_feats\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx]) #categorical_feature=categorical_feats\n\n    iteration = 2000\n    lgb_m = lgb.train(params, trn_data, iteration, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 100)\n    fold_pred[val_idx] = lgb_m.predict(train.iloc[val_idx], num_iteration=lgb_m.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = train.columns\n    fold_importance_df[\"importance\"] = lgb_m.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    lgb_preds += lgb_m.predict(test, num_iteration=lgb_m.best_iteration) / (nfolds*nrepeats)\n\nprint(\"CV score: {:<8.5f}\".format(np.sqrt(mean_squared_error(fold_pred, target))))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffe7c71f66445c8fa5b8077326c2f880dfcf453a"},"cell_type":"code","source":"# ranking all feature by avg importance score from Kfold, select top100\nall_features = feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)\nall_features.reset_index(inplace=True)\nimportant_features = list(all_features[0:100]['feature'])\nall_features[0:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2091fa0e9b91d0a47e739238e23d6d961b93a383"},"cell_type":"code","source":"# Check feature correlation \n# important_features = list(final_importance['feature'][0:60])\ndf = train[important_features]\ncorr_matrix = df.corr().abs()\n\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n\n# Find index of feature columns with correlation greater than 0.95\nhigh_cor = [column for column in upper.columns if any(upper[column] > 0.95)]\nprint(len(high_cor))\nprint(high_cor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bdb72836d81b7c041345ad73b1652a985de19ac"},"cell_type":"code","source":"# final selected features: drop highly correlated features from important features.\nfeatures = [i for i in important_features if i not in high_cor]\nprint(len(features))\nprint(features)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d97fdef1e722c9375286bc63f4568cf1fad29ca0"},"cell_type":"markdown","source":"## 2nd round: Train model with selected important_features only"},{"metadata":{"trusted":true,"_uuid":"62637b85b24579d2873eed03221b83d22c32a95c"},"cell_type":"code","source":"# params for 2nd round training\nparams = {'boosting': 'gbdt',\n          'objective':'regression',\n          'metric': 'rmse',\n          'learning_rate': 0.003, # 0.003! #0.005 #0.006 \n          'num_leaves': 110, #110 #100 #150 large, but over-fitting\n          'max_bin': 66,  #60 #50 # large,but slower,over-fitting\n          'max_depth': 10, # deal with over-fitting\n          'min_data_in_leaf': 30, # deal with over-fitting\n          'min_child_samples': 20,\n          'feature_fraction': 0.8,#0.5 #0.6 #0.8\n          'bagging_fraction': 0.8,\n          'bagging_freq': 40,#5  \n          'bagging_seed': 11,\n          'lambda_l1': 2,#1.3! #5 #1.2 #1\n          'lambda_l2': 0.1 #0.1\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f84de8e071a0158a0e6f8df0c6fef2702358923f"},"cell_type":"code","source":"train = train[features]\ntest = test[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"641844a43131b8c77c931cc7198e353c9abd28cd"},"cell_type":"code","source":"# Use Kfold predict\nnfolds = 5\nnrepeats = 2 \n\nfolds = RepeatedKFold(n_splits=nfolds, n_repeats=nrepeats, random_state=11)\nfold_pred = np.zeros(len(train))\nlgb_preds = np.zeros(len(test))\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)): #target.values\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx]) #categorical_feature=categorical_feats\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx]) #categorical_feature=categorical_feats\n\n    iteration = 3000\n    lgb_model = lgb.train(params, trn_data, iteration, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 100)\n    fold_pred[val_idx] = lgb_model.predict(train.iloc[val_idx], num_iteration=lgb_model.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = train.columns\n    fold_importance_df[\"importance\"] = lgb_model.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    lgb_preds += lgb_model.predict(test, num_iteration=lgb_model.best_iteration) / (nfolds*nrepeats)\n\nprint(\"CV score: {:<8.5f}\".format(np.sqrt(mean_squared_error(fold_pred, target))))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96af73cd04fa5f4c72f8235a42d7d10332c14e99"},"cell_type":"code","source":"# training data label \ntarget.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c84a5d94354e803e8dbb60373f0e152fe7de4ab"},"cell_type":"code","source":"# predicted values\npd.DataFrame(lgb_preds).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e97ae28515627e0002e528542a2e2607a97b691"},"cell_type":"code","source":"# predicted value distribution\nimport matplotlib.pyplot as plt\n\nnum_bins = 100\nn, bins, patches = plt.hist(lgb_preds, num_bins, facecolor='blue', alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40097b8507866ea171ab4a2dfad0a6b42341df0e"},"cell_type":"code","source":"# Add target value to submition file\nlgb_submition[\"target\"] = lgb_preds\nlgb_submition.to_csv(\"lgb_submition.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a83961ac3fb3b8fa4c332676e5b498a0c0715d48"},"cell_type":"code","source":"# feature importance\nfinal_importance = feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)\nfinal_importance.reset_index(inplace=True)\nfinal_importance[0:50]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ecdb29aecd2061d4d2468eba554a8270caa1cf7"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",y=\"feature\",data=final_importance)\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}