{"cells":[{"metadata":{"_uuid":"a9ef20fd4e6471fd2c9e3022edfdddc07a970aca"},"cell_type":"markdown","source":"## P.S. The main idea behind this notebook is inspired from FabienDaniel Kernel Elo_world.\nhttps://www.kaggle.com/fabiendaniel/elo-world"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nimport warnings\nimport time\nimport sys\nimport datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import BayesianRidge\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50e1f88df96b5e525237fe120650e679b7f60654"},"cell_type":"code","source":"new_transactions = pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv', parse_dates=['purchase_date'])\nhistorical_transactions = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv', parse_dates=['purchase_date'])\n\ndef binarize(df):\n    for col in ['authorized_flag', 'category_1']:\n        df[col] = df[col].map({'Y':1, 'N':0})\n    return df\n\nhistorical_transactions = binarize(historical_transactions)\nnew_transactions = binarize(new_transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a940a9ce940fa489a5a5255f6d4523c9b7c3ffdc"},"cell_type":"code","source":"%%time\ndef read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    return df\n\ntrain = read_data('../input/elo-merchant-category-recommendation/train.csv')\ntest = read_data('../input/elo-merchant-category-recommendation/test.csv')\n\ntarget = train['target']\ndel train['target']\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd9f71f39664eef55e0b3ec3e6172e84a8e1a963"},"cell_type":"markdown","source":"## Feature Engineering"},{"metadata":{"trusted":true,"_uuid":"170f0e959ba5250b191b7cee6b586bdabd347018"},"cell_type":"code","source":"%%time\nhistorical_transactions = pd.get_dummies(historical_transactions, columns=['category_2', 'category_3'])\nnew_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])\n\nhistorical_transactions = reduce_mem_usage(historical_transactions)\nnew_transactions = reduce_mem_usage(new_transactions)\n\nagg_fun = {'authorized_flag': ['sum', 'mean']}\nauth_mean = historical_transactions.groupby(['card_id']).agg(agg_fun)\nauth_mean.columns = ['_'.join(col).strip() for col in auth_mean.columns.values]\nauth_mean.reset_index(inplace=True)\n\nauthorized_transactions = historical_transactions[historical_transactions['authorized_flag'] == 1]\nhistorical_transactions = historical_transactions[historical_transactions['authorized_flag'] == 0]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90536f89e81fc4bb7f45a17f47e4b627f53e69b4"},"cell_type":"code","source":"%%time\nhistorical_transactions['purchase_month'] = historical_transactions['purchase_date'].dt.month\nauthorized_transactions['purchase_month'] = authorized_transactions['purchase_date'].dt.month\nnew_transactions['purchase_month'] = new_transactions['purchase_date'].dt.month\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5de9c3bcc2ad2dfeb943e1b02fef3e433c91af3"},"cell_type":"code","source":"%%time\ndef aggregate_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'category_1': ['sum', 'mean'],\n        'category_2_1.0': ['mean'],\n        'category_2_2.0': ['mean'],\n        'category_2_3.0': ['mean'],\n        'category_2_4.0': ['mean'],\n        'category_2_5.0': ['mean'],\n        'category_3_A': ['mean'],\n        'category_3_B': ['mean'],\n        'category_3_C': ['mean'],\n        'merchant_id': ['nunique'],\n        'merchant_category_id': ['nunique'],\n        'state_id': ['nunique'],\n        'city_id': ['nunique'],\n        'subsector_id': ['nunique'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std'],\n        'purchase_month': ['mean', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp, 'min', 'max'],\n        'month_lag': ['min', 'max']\n        }\n    \n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf1f963f61841ef34f79fd16364bbf59b7f4e8a4"},"cell_type":"code","source":"%%time\nhistory = aggregate_transactions(historical_transactions)\nhistory.columns = ['hist_' + c if c != 'card_id' else c for c in history.columns]\nhistory[:5]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c1bdbaa7c16e22436893354b31d47e1846330ab"},"cell_type":"code","source":"%%time\nauthorized = aggregate_transactions(authorized_transactions)\nauthorized.columns = ['auth_' + c if c != 'card_id' else c for c in authorized.columns]\nauthorized[:5]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cc2775c29c17cb7438f3f5c45777b894afa94fc"},"cell_type":"code","source":"%%time\nnew = aggregate_transactions(new_transactions)\nnew.columns = ['new_' + c if c != 'card_id' else c for c in new.columns]\nnew[:5]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb443fb1e2a47f5a0bef02921a2cba1d4e95e673"},"cell_type":"code","source":"%%time\ndef aggregate_per_month(history):\n    grouped = history.groupby(['card_id', 'month_lag'])\n\n    agg_func = {\n            'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n            'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n            }\n\n    intermediate_group = grouped.agg(agg_func)\n    intermediate_group.columns = ['_'.join(col).strip() for col in intermediate_group.columns.values]\n    intermediate_group.reset_index(inplace=True)\n\n    final_group = intermediate_group.groupby('card_id').agg(['mean', 'std'])\n    final_group.columns = ['_'.join(col).strip() for col in final_group.columns.values]\n    final_group.reset_index(inplace=True)\n    \n    return final_group\n#___________________________________________________________\nfinal_group =  aggregate_per_month(historical_transactions) \nfinal_group[:10]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e5e313f605160783127a6f0798f176eaf6a55a4"},"cell_type":"code","source":"%%time\ntrain = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')\n\ntrain = pd.merge(train, authorized, on='card_id', how='left')\ntest = pd.merge(test, authorized, on='card_id', how='left')\n\ntrain = pd.merge(train, new, on='card_id', how='left')\ntest = pd.merge(test, new, on='card_id', how='left')\n\ntrain = pd.merge(train, final_group, on='card_id', how='left')\ntest = pd.merge(test, final_group, on='card_id', how='left')\n\ntrain = pd.merge(train, auth_mean, on='card_id', how='left')\ntest = pd.merge(test, auth_mean, on='card_id', how='left')\n\nprint(\"Train Shape:\", train.shape)\nprint(\"Test Shape:\", test.shape)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"796d4e17b4c4cc3121fc833669fc2eaf07bf4fba"},"cell_type":"code","source":"features = [c for c in train.columns if c not in ['card_id', 'first_active_month']]\ncategorical_feats = [c for c in features if 'feature_' in c]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b926c1b3a703d8ed6c7c609e7c892b503e3c511"},"cell_type":"markdown","source":"## LightGBM"},{"metadata":{"trusted":true,"_uuid":"28ac4d7abc8117671e8e79caaabcbf41dbb328c0"},"cell_type":"code","source":"param = {'num_leaves': 31,\n         'min_data_in_leaf': 32, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.005,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"nthread\": 4,\n         \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92bf1d3418327cb759937b8f32f40d75e73108e9","scrolled":false},"cell_type":"code","source":"folds = KFold(n_splits=5, shuffle=True, random_state=15)\noof = np.zeros(len(train))\npredictions = np.zeros(len(test))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n\n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)\n    oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test[features], num_iteration=clf.best_iteration) / folds.n_splits\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98eb235d712c857ebc3dc3fff312e049c811267a","scrolled":false},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"acdaccc5c862736b097ae9e5bcdcdae5083b5d33"},"cell_type":"markdown","source":"## LightGBM-1 with Repeated kfold approach\n\n#### RepeatedKFold repeats K-Fold n times. It can be used when one requires to run KFold n times, producing different splits in each repetition."},{"metadata":{"trusted":true,"_uuid":"91043a8b0a7b1d335672f35d39f1621b02befc88"},"cell_type":"code","source":"lgbparam = {'num_leaves': 31,\n            'boosting_type': 'rf',\n             'min_data_in_leaf': 30, \n             'objective':'regression',\n             'max_depth': -1,\n             'learning_rate': 0.01,\n             \"min_child_samples\": 20,\n             \"boosting\": \"gbdt\",\n             \"feature_fraction\": 0.9,\n             \"bagging_freq\": 1,\n             \"bagging_fraction\": 0.9 ,\n             \"bagging_seed\": 11,\n             \"metric\": 'rmse',\n             \"lambda_l1\": 0.1,\n             \"verbosity\": -1,\n             \"nthread\": 4,\n             \"random_state\": 4590}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"17b17b5d967ccf2912d9f7b229cd06cb74819d85"},"cell_type":"code","source":"from sklearn.model_selection import RepeatedKFold\nfolds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4520)\n\noof_lgb = np.zeros(len(train))\npredictions_lgb = np.zeros(len(test))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n\n    num_round = 11000\n    clf = lgb.train(lgbparam, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 100)\n    oof_lgb[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions_lgb += clf.predict(test[features], num_iteration=clf.best_iteration) / (5 * 2)\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof_lgb, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b84242fc4c57f96b36b860d5728cddaf3b8c2943"},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0362877a4fb45006383dc3bc03b5642aba0f47dc"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test[\"card_id\"].values})\nsub_df[\"target\"] = predictions\nsub_df.to_csv(\"submit_lgb.csv\", index=False)\n\nsub_df1 = pd.DataFrame({\"card_id\":test[\"card_id\"].values})\nsub_df1[\"target\"] = predictions_lgb\nsub_df1.to_csv(\"submit_lgb1.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2dfa7a5fcabd78829678fa4283e99862f1c4a0d7"},"cell_type":"markdown","source":"## Stacking"},{"metadata":{"trusted":true,"_uuid":"8c9afce8b59b5f51024150a997a9d2eb702ab85d"},"cell_type":"code","source":"train_stack = np.vstack([oof,oof_lgb]).transpose()\ntest_stack = np.vstack([predictions,predictions_lgb]).transpose()\n\nfolds = RepeatedKFold(n_splits=5,n_repeats=1,random_state=4520)\noof_stack = np.zeros(train_stack.shape[0])\npredictions_stack = np.zeros(test_stack.shape[0])\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_stack, target)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values\n    val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values\n\n    print(\"-\" * 10 + \"Stacking \" + str(fold_) + \"-\" * 10)\n#     cb_model = CatBoostRegressor(iterations=3000, learning_rate=0.1, depth=8, l2_leaf_reg=20, bootstrap_type='Bernoulli',  eval_metric='RMSE', metric_period=50, od_type='Iter', od_wait=45, random_seed=17, allow_writing_files=False)\n#     cb_model.fit(trn_data, trn_y, eval_set=(val_data, val_y), cat_features=[], use_best_model=True, verbose=True)\n    clf = BayesianRidge()\n    clf.fit(trn_data, trn_y)\n    \n    oof_stack[val_idx] = clf.predict(val_data)\n    predictions_stack += clf.predict(test_stack) / 5\n\n\nnp.sqrt(mean_squared_error(target.values, oof_stack))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e871862de58a3ce43c148f78f152c1d6385a0a08"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv')\nsample_submission['target'] = predictions_stack\nsample_submission.to_csv('Bayesian_Ridge_Stacking.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b65e47686b49b31f0dc47171ab2aece4a4d0a941"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv')\nsample1 = pd.read_csv(\"../input/elo-blending/3.695.csv\")\nsample2 = pd.read_csv(\"../input/elo-blending/combining_submission (1).csv\")\nsample_submission['target'] = predictions * 0.5 + predictions_lgb * 0.5\nsample_submission.to_csv(\"Blend1.csv\", index = False)\nsample_submission['target'] = sample_submission['target'] * 0.2 + sample1['target'] * 0.2 + sample2['target'] * 0.6\nsample_submission.to_csv('Blend2.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}