{"cells":[{"metadata":{"_uuid":"524f74a283e94af3fe8299a5f3bd1b58eeacd2b2"},"cell_type":"markdown","source":"In this kernel we'll use the Bayesian Hyperparameter Tuning to find the optimum hyperparameters for LightGBM. This is the first kernel I'm writing so please let me know how I can improve. :)\n\nI used the following Kernels and blogs for reference\n\n* [Simple Exploration Notebook - Elo](https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-elo)\n* [Simple Bayesian Optimization for LightGBM](https://www.kaggle.com/sz8416/simple-bayesian-optimization-for-lightgbm)\n* [A Conceptual Explanation of Bayesian Hyperparameter Optimization for Machine Learning](https://towardsdatascience.com/a-conceptual-explanation-of-bayesian-model-based-hyperparameter-optimization-for-machine-learning-b8172278050f)\n* [Automated Machine Learning Hyperparameter Tuning in Python](https://towardsdatascience.com/automated-machine-learning-hyperparameter-tuning-in-python-dfda59b72f8a)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nfrom sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\nfrom bayes_opt import BayesianOptimization\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b80cd7ae160231cb08f85b75a8e1743ed5198d98"},"cell_type":"markdown","source":"## Quick Feature Engineering"},{"metadata":{"trusted":true,"_uuid":"bfdab255f309f4ca9239eaa07f17398e643010a3"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest_df = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebb27ee19be1dc40b94731d4dcd0ce670aacc86d"},"cell_type":"code","source":"hist_df = pd.read_csv(\"../input/historical_transactions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"756c8ef8d3756fb21365e0a66540b7eb92a0b5d3"},"cell_type":"code","source":"gdf = hist_df.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].size().reset_index()\ngdf.columns = [\"card_id\", \"num_hist_transactions\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"216d9ff997e3cafaf1be538f5a7227eabc6fa9af"},"cell_type":"code","source":"gdf = hist_df.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].agg(['sum', 'mean', 'std', 'min', 'max']).reset_index()\ngdf.columns = [\"card_id\", \"sum_hist_trans\", \"mean_hist_trans\", \"std_hist_trans\", \"min_hist_trans\", \"max_hist_trans\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"191aed6554c8698d5a0a4b782fe1ed4e5a306a8a"},"cell_type":"code","source":"new_trans_df = pd.read_csv(\"../input/new_merchant_transactions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"e2c425d22c6100926bf1ca200271e80a4c86ec34"},"cell_type":"code","source":"gdf = new_trans_df.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].size().reset_index()\ngdf.columns = [\"card_id\", \"num_merch_transactions\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b860890fa58461a623b3706e8f1509b33902c660"},"cell_type":"code","source":"gdf = new_trans_df.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].agg(['sum', 'mean', 'std', 'min', 'max']).reset_index()\ngdf.columns = [\"card_id\", \"sum_merch_trans\", \"mean_merch_trans\", \"std_merch_trans\", \"min_merch_trans\", \"max_merch_trans\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5471ee15f494ad6977a89eef7c12f130622099aa"},"cell_type":"code","source":"train_df[\"year\"] = train_df[\"first_active_month\"].dt.year\ntest_df[\"year\"] = test_df[\"first_active_month\"].dt.year\ntrain_df[\"month\"] = train_df[\"first_active_month\"].dt.month\ntest_df[\"month\"] = test_df[\"first_active_month\"].dt.month\n\ncols_to_use = [\"feature_1\", \"feature_2\", \"feature_3\", \"year\", \"month\", \n               \"num_hist_transactions\", \"sum_hist_trans\", \"mean_hist_trans\", \"std_hist_trans\", \n               \"min_hist_trans\", \"max_hist_trans\",\n               \"num_merch_transactions\", \"sum_merch_trans\", \"mean_merch_trans\", \"std_merch_trans\",\n               \"min_merch_trans\", \"max_merch_trans\",\n              ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"663e8744edb14d7a66fa9d2a59017dcd3ee5b00e"},"cell_type":"code","source":"train_X = train_df[cols_to_use]\ntest_X = test_df[cols_to_use]\ntrain_y = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d18bdb10c965d148a1418774b68a24637ca7e8f"},"cell_type":"markdown","source":"## Hyperparameter Tuning using Bayesian Hyperparameter Optimization "},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e031a242e3d2dd9cfe91bf978ad1ef09185c74fc"},"cell_type":"code","source":"def bayes_parameter_opt_lgb(X, y, init_round=20, opt_round=30, n_folds=5, random_seed=6, n_estimators=10000,\n                            learning_rate=0.05, output_process=False):\n    # prepare data\n\n    train_data = lgb.Dataset(data=X, label=y)\n    # parameters\n\n    def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth, \n                 lambda_l1, lambda_l2, min_split_gain, min_child_weight):\n\n        params = {'objective':'regression','num_iterations':1000, 'learning_rate':0.05,\n                  'early_stopping_round':100, 'metric':'rmse'}\n        params[\"num_leaves\"] = int(round(num_leaves))\n        params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n        params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n        params['max_depth'] = int(round(max_depth))\n        params['lambda_l1'] = max(lambda_l1, 0)\n        params['lambda_l2'] = max(lambda_l2, 0)\n        params['min_split_gain'] = min_split_gain\n        params['min_child_weight'] = min_child_weight\n        \n        cv_result = lgb.cv(params, train_data, nfold=3, seed=random_seed,\n                           stratified=False, verbose_eval =200, metrics=['rmse'])\n\n        return min(cv_result['rmse-mean'])\n\n    # setting range of the parameters\n    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (24, 45),\n                                            'feature_fraction': (0.1, 0.9),\n                                            'bagging_fraction': (0.5, 1),\n                                            'max_depth': (5, 8.99),\n                                            'lambda_l1': (0, 5),\n                                            'lambda_l2': (0, 3),\n                                            'min_split_gain': (0.001, 0.1),\n                                            'min_child_weight': (5, 60)}, random_state=0)\n    # optimize\n    lgbBO.maximize(init_points=init_round, n_iter=opt_round)\n    \n    # output optimization process\n    if output_process==True: lgbBO.points_to_csv(\"bayes_opt_result.csv\")\n    \n    # return\n    return lgbBO\n\nopt_params = bayes_parameter_opt_lgb(train_X, train_y, init_round=5, opt_round=10, n_folds=3,\n                                     random_seed=6, n_estimators=10000, learning_rate=0.05)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c07c52028a88462e4f5af821c56dd71ffc976510"},"cell_type":"code","source":"params = opt_params.max['params']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e21ac610ca4c8121ff6584c29b73f9f42f572419"},"cell_type":"markdown","source":"## Params that were optimum"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"380f94732032dcf0c31e71bbb4fc2152cc3baa8e"},"cell_type":"code","source":"params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99d992ac4706710f770dbc6ad20f83b832181f84"},"cell_type":"code","source":"params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\",\n    \"bagging_frequency\" : 5,\n    \"bagging_seed\" : 2018,\n    \"verbosity\" : -1,\n\n    # Selected rounded-off params\n    'bagging_fraction': 0.7,\n    'feature_fraction': 0.1,\n    'lambda_l1': 1,\n    'lambda_l2': 0,\n    'max_depth': 9,\n    'min_child_weight': 5,\n    'min_split_gain': 0,\n    'num_leaves': 24\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"100e564d6f6afcd7da8445496cd4b0d0b91946ba"},"cell_type":"code","source":"params","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"795815680adf09f646f180636d05bb526490eb8a"},"cell_type":"markdown","source":"## Training LightGBM using our newly learned params."},{"metadata":{"trusted":true,"_uuid":"79889143bc5817f70245549d73365fc8b162b105"},"cell_type":"code","source":"def run_lgb(train_X, train_y, val_X, val_y, test_X):\n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    evals_result = {}\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, \n                      verbose_eval=100, evals_result=evals_result)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, evals_result\n\npred_test = 0\nkf = model_selection.KFold(n_splits=5, random_state=2018, shuffle=True)\nfor dev_index, val_index in kf.split(train_df):\n    dev_X, val_X = train_X.loc[dev_index,:], train_X.loc[val_index,:]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    \n    pred_test_tmp, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)\n    pred_test += pred_test_tmp\npred_test /= 5.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6479cb3ed15c5556045d2fc048c56d614bee3a5"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,10))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7371f1139137edebc6a475d2f3bffcbcdf6c1b00"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test_df[\"card_id\"].values})\nsub_df[\"target\"] = pred_test\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}