{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import KFold\nfrom sklearn import model_selection, preprocessing, metrics\n\nimport warnings\nimport datetime\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Loading Train and Test Data\ndf_train = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ndf_test = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])\nprint(\"{} observations and {} features in train set.\".format(df_train.shape[0],df_train.shape[1]))\nprint(\"{} observations and {} features in test set.\".format(df_test.shape[0],df_test.shape[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a9ba09a9ccf12cfc12c7f8140ef8fd13b42e2af"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21a54a99faee8995716d84ce3a5104fc5913fe5b"},"cell_type":"code","source":"df_train[\"month\"] = df_train[\"first_active_month\"].dt.month\ndf_test[\"month\"] = df_test[\"first_active_month\"].dt.month\ndf_train[\"year\"] = df_train[\"first_active_month\"].dt.year\ndf_test[\"year\"] = df_test[\"first_active_month\"].dt.year\ndf_train['elapsed_time'] = (datetime.date(2018, 2, 1) - df_train['first_active_month'].dt.date).dt.days\ndf_test['elapsed_time'] = (datetime.date(2018, 2, 1) - df_test['first_active_month'].dt.date).dt.days\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1625d96b28dfad8f50498cdb9ae43aa7b71ecf81"},"cell_type":"code","source":"df_train = pd.get_dummies(df_train, columns=['feature_1', 'feature_2'])\ndf_test = pd.get_dummies(df_test, columns=['feature_1', 'feature_2'])\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65a46dfb7dca1df3470cc913e91db59acd375c00"},"cell_type":"code","source":"df_hist_trans = pd.read_csv(\"../input/historical_transactions.csv\")\ndf_hist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5105d495cca8b9e5b71565b6bcc277709661fe9"},"cell_type":"code","source":"df_hist_trans = pd.get_dummies(df_hist_trans, columns=['category_2', 'category_3'])\ndf_hist_trans['authorized_flag'] = df_hist_trans['authorized_flag'].map({'Y': 1, 'N': 0})\ndf_hist_trans['category_1'] = df_hist_trans['category_1'].map({'Y': 1, 'N': 0})\ndf_hist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10db1108366af0d94676fa9078f7c5be28030983"},"cell_type":"code","source":"def aggregate_transactions(trans, prefix):  \n    trans.loc[:, 'purchase_date'] = pd.DatetimeIndex(trans['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean', 'median'],\n        'category_1': ['mean', 'median'],\n        'category_2_1.0': ['mean', 'median'],\n        'category_2_2.0': ['mean', 'median'],\n        'category_2_3.0': ['mean', 'median'],\n        'category_2_4.0': ['mean', 'median'],\n        'category_2_5.0': ['mean', 'median'],\n        'category_3_A': ['mean', 'median'],\n        'category_3_B': ['mean', 'median'],\n        'category_3_C': ['mean', 'median'],\n        'merchant_id': ['nunique'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std', 'median'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std', 'median'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max']\n    }\n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d247ff2981a60c76305bd5dc6cef475c58bc5277"},"cell_type":"code","source":"import gc\nmerch_hist = aggregate_transactions(df_hist_trans, prefix='hist_')\ndel df_hist_trans\ngc.collect()\ndf_train = pd.merge(df_train, merch_hist, on='card_id',how='left')\ndf_test = pd.merge(df_test, merch_hist, on='card_id',how='left')\ndel merch_hist\ngc.collect()\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a976d1f49ac6e141896fc665d9749ce0a86273f9"},"cell_type":"code","source":"df_new_trans = pd.read_csv(\"../input/new_merchant_transactions.csv\")\ndf_new_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52796d28c02319492b8cce4833d146e207c6ab31"},"cell_type":"code","source":"df_new_trans = pd.get_dummies(df_new_trans, columns=['category_2', 'category_3'])\ndf_new_trans['authorized_flag'] = df_new_trans['authorized_flag'].map({'Y': 1, 'N': 0})\ndf_new_trans['category_1'] = df_new_trans['category_1'].map({'Y': 1, 'N': 0})\ndf_new_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f3f5ea392cd6dfdf3ea37beff28117489be73b5"},"cell_type":"code","source":"merch_new = aggregate_transactions(df_new_trans, prefix='new_')\ndel df_new_trans\ngc.collect()\ndf_train = pd.merge(df_train, merch_new, on='card_id',how='left')\ndf_test = pd.merge(df_test, merch_new, on='card_id',how='left')\ndel merch_new\ngc.collect()\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0004beb9b30965b9bec3ab87e5199ec0847296ca"},"cell_type":"code","source":"# TO DO add merchants","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06d1bb53d634a8ad34460f6ace7670c4f3247a5c"},"cell_type":"code","source":"target = df_train['target']\ndrops = ['card_id', 'first_active_month', 'target']\nuse_cols = [c for c in df_train.columns if c not in drops]\nfeatures = list(df_train[use_cols].columns)\ndf_train[features].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7fa2ee514046434028d49971165fc796e5cacd4"},"cell_type":"code","source":"print(df_train[features].shape)\nprint(df_test[features].shape) # Validation set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf4ff23e322814531bb39213e50f070a5c69be0b"},"cell_type":"code","source":"import contextlib\nimport xgboost as xgb\nfrom bayes_opt import BayesianOptimization\nfrom sklearn.metrics import mean_squared_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12c9fd3648f67b6b4e9e24579e18bb7d044271ac"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(df_train[features],\n                                                    target, test_size=0.2)\ndtrain = xgb.DMatrix(X_train, label=y_train)\ndtest = xgb.DMatrix(X_test)\n\ndel(df_train)\ndel(X_train)\ndel(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce114f724b534628fa85799f0555b4fe59e4b30b"},"cell_type":"code","source":"dholdout = xgb.DMatrix(df_test[features])\ntarget_holdout = df_test[\"card_id\"].values\n\ndel(df_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5517166375bd140f4fa03cfca86af578edfc9879"},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true,"_uuid":"af6a41b5c4413662673116fea82b018390c3ce83"},"cell_type":"code","source":"# # https://www.kaggle.com/tilii7/bayesian-optimization-of-xgboost-parameters/notebook\n# def xgb_evaluate(max_depth, \n#                  gamma,\n#                  min_child_weight,\n#                  max_delta_step,\n#                  subsample,\n#                  colsample_bytree):\n    \n#     global RMSEbest\n#     global ITERbest\n    \n#     params = {'eval_metric': 'rmse',\n#               'nthread' : 4,\n#               'silent' : True,\n#               'max_depth': int(max_depth),\n#               'subsample': max(min(subsample, 1), 0),\n#               'eta': 0.1,\n#               'gamma': gamma,\n#               'colsample_bytree': max(min(colsample_bytree, 1), 0),   \n#               'min_child_weight': min_child_weight ,\n#               'max_delta_step':int(max_delta_step),\n#               'seed' : 16\n#              }\n    \n#     folds = 5\n#     cv_score = 0\n    \n#     print(\"\\n Search parameters (%d-fold validation):\\n %s\" % (folds, params), file=log_file )\n#     log_file.flush()\n    \n#     # Used around 1000 boosting rounds in the full model\n#     cv_result = xgb.cv(params, \n#                        dtrain, \n#                        num_boost_round=2000, # can be increased 1k ~ 20k\n#                        nfold=folds,\n# #                       verbose_eval = 10,\n#                        early_stopping_rounds = 100,\n#                        metrics = 'rmse',\n#                        show_stdv = True\n#                       )    \n    \n#     val_score = cv_result['test-rmse-mean'].iloc[-1]\n#     train_score = cv_result['train-rmse-mean'].iloc[-1]\n#     print(' Stopped after %d iterations with train-rmse = %f val-rmse = %f ( diff = %f ) ' % ( len(cv_result), train_score, val_score, (train_score - val_score) ) )\n#     if ( val_score > RMSEbest ):\n#         RMSEbest = val_score\n#         ITERbest = len(cv_result)\n\n#     # Bayesian optimization only knows how to maximize, not minimize, so return the negative RMSE\n#     return -1.0 * val_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"426750956dafdf4f796049f269f900597366de5a"},"cell_type":"code","source":"# log_file = open('Elo-RMSE-5fold-XGB-run-01-v1-full.log', 'a')\n# RMSEbest = -1.\n# ITERbest = 0\n\n# xgb_bo = BayesianOptimization(xgb_evaluate, {\n#                                     'max_depth': (2, 12),\n#                                      'gamma': (0.001, 10.0),\n#                                      'min_child_weight': (0, 20),\n#                                      'max_delta_step': (0, 10),\n#                                      'subsample': (0.4, 1.0),\n#                                      'colsample_bytree' :(0.4, 1.0)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8e82b49104263569b8c9d7fa4d4a27d4ce4fce0"},"cell_type":"code","source":"# xgb_bo.explore({\n#               'max_depth':            [3, 8, 3, 8, 8, 3, 8, 3],\n#               'gamma':                [0.5, 8, 0.2, 9, 0.5, 8, 0.2, 9],\n#               'min_child_weight':     [0.2, 0.2, 0.2, 0.2, 12, 12, 12, 12],\n#               'max_delta_step':       [1, 2, 2, 1, 2, 1, 1, 2],\n#               'subsample':            [0.6, 0.8, 0.6, 0.8, 0.6, 0.8, 0.6, 0.8],\n#               'colsample_bytree':     [0.6, 0.8, 0.6, 0.8, 0.6, 0.8, 0.6, 0.8],\n#               })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35e01dcc99a1cff6ed8c460aac554942ff9fff95"},"cell_type":"code","source":"# %%time\n\n# # Use the expected improvement acquisition function to handle negative numbers\n# # Optimally needs quite a few more initiation points 15-20 and number of iterations 25-50\n\n# print('-'*130)\n# print('-'*130, file=log_file)\n# log_file.flush()\n\n# with warnings.catch_warnings():\n#     warnings.filterwarnings('ignore')\n#     #xgb_bo.maximize(init_points=1, n_iter=1, acq='ei', xi=0.0)\n#     xgb_bo.maximize(init_points=10, n_iter=50, acq='ei', xi=0.01)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"570ffffa7bf5f8f9013a0fe6d43489587fa7550d"},"cell_type":"markdown","source":"Best model"},{"metadata":{"trusted":true,"_uuid":"495f6b0c32fb223942343a01f29433518dacd45c"},"cell_type":"code","source":"# print('-'*130)\n# print('Final Results')\n# print('Maximum XGBOOST value: %f' % xgb_bo.res['max']['max_val'])\n# print('Best XGBOOST parameters: ', xgb_bo.res['max']['max_params'])\n# print('-'*130, file=log_file)\n# print('Final Result:', file=log_file)\n# print('Maximum XGBOOST value: %f' % xgb_bo.res['max']['max_val'], file=log_file)\n# print('Best XGBOOST parameters: ', xgb_bo.res['max']['max_params'], file=log_file)\n# log_file.flush()\n# log_file.close()\n\n# history_df = pd.DataFrame(xgb_bo.res['all']['params'])\n# history_df2 = pd.DataFrame(xgb_bo.res['all']['values'])\n# history_df = pd.concat((history_df, history_df2), axis=1)\n# history_df.rename(columns = { 0 : 'rmse'}, inplace=True)\n# history_df['rmse'] = np.abs(history_df['rmse'])\n# history_df.to_csv('Elo-RMSE-5fold-XGB-run-01-v1-grid.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab00e237917dac8088e04e719bd8b9891d2b495d"},"cell_type":"code","source":"# pd.read_csv('Elo-RMSE-5fold-XGB-run-01-v1-grid.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15c7c0352a3757c98377fcada4c07d6a797d700c"},"cell_type":"code","source":"params1 = {'max_depth': 12.0, \n          'gamma': 3.1390339637040787, \n          'min_child_weight': 0.0, \n          'max_delta_step': 10.0, \n          'subsample': 1.0, \n          'colsample_bytree': 1.0}\n\nparams2 = {'colsample_bytree': 0.4,\n          'gamma': 7.438375302732893,\n          'max_delta_step': 3.0732617140069647,\n          'max_depth': 9.926188389437563,\n          'min_child_weight': 20.0,\n          'subsample': 1.0}\n\nparams1['max_depth'] = int(params1['max_depth'])\nparams2['max_depth'] = int(params2['max_depth'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec6b476d4b880167f6b7ccad91bdd032dd4f9419"},"cell_type":"markdown","source":"# Testing"},{"metadata":{"trusted":true,"_uuid":"cf9f75d2b9a39cc54849a2cfdd810ba0bdfcc4a7"},"cell_type":"code","source":"%%time \n\n# Train a new model with the best parameters from the search\nmodel1 = xgb.train(params1, dtrain, num_boost_round=500)\nmodel2 = xgb.train(params2, dtrain, num_boost_round=500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bb90831c3ef3eceb60c06c50aeaba72d3444baf"},"cell_type":"code","source":"# # Predict on testing and training set\n# y_pred = model2.predict(dtest)\n# y_train_pred = model2.predict(dtrain)\n\n# # Report testing and training RMSE\n# print('Test error:', np.sqrt(mean_squared_error(y_test, y_pred)))\n# print('Train error:', np.sqrt(mean_squared_error(y_train, y_train_pred)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"317af715dd6f2614ad4da4d295481933ef6db23a"},"cell_type":"markdown","source":"# Feature importance"},{"metadata":{"trusted":true,"_uuid":"d199ae1c8666918f6e15221c061847dae2d8e7fc"},"cell_type":"code","source":"# # feature importance\n# fig =  plt.figure(figsize = (12,8))\n# axes = fig.add_subplot(111)\n# xgb.plot_importance(model2,ax = axes,height =0.5)\n# sns.despine()\n# plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"393241352c30b5276e40c1791cd396b87779eb0e"},"cell_type":"markdown","source":"# Prediction "},{"metadata":{"trusted":true,"_uuid":"b07dbe9c66e8d1dac93bc71185335a2ceb01af3a"},"cell_type":"code","source":"predictions1 = model1.predict(dholdout)\npredictions2 = model2.predict(dholdout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cb9230475eeada523a3b2343827f44307e9fa9f"},"cell_type":"code","source":"df_sub = pd.DataFrame({'card_id': target_holdout, 'target': predictions1})\ndf_sub.to_csv('p1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9246791e7bc123faf6bce511ea764b0764d867ae"},"cell_type":"code","source":"df_sub = pd.DataFrame({'card_id': target_holdout, 'target': predictions2})\ndf_sub.to_csv('p2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"443ee0fa6d2018e4b91547e44199301f6a774ec0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}