{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"id":"ZgSfUUo9uWNq","execution":{"iopub.status.busy":"2022-04-05T19:34:27.587186Z","iopub.execute_input":"2022-04-05T19:34:27.587488Z","iopub.status.idle":"2022-04-05T19:34:27.595024Z","shell.execute_reply.started":"2022-04-05T19:34:27.587446Z","shell.execute_reply":"2022-04-05T19:34:27.594433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install optuna","metadata":{"executionInfo":{"elapsed":11947,"status":"ok","timestamp":1648625925604,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"},"user_tz":-120},"id":"1WBj1l5pcozv","outputId":"85490be3-63ad-45d5-b8e6-bbcbafe98e16","execution":{"iopub.status.busy":"2022-04-05T19:34:27.724952Z","iopub.execute_input":"2022-04-05T19:34:27.725394Z","iopub.status.idle":"2022-04-05T19:34:27.728528Z","shell.execute_reply.started":"2022-04-05T19:34:27.725366Z","shell.execute_reply":"2022-04-05T19:34:27.727898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.ensemble import AdaBoostClassifier, HistGradientBoostingClassifier, GradientBoostingClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\nfrom sklearn.metrics import accuracy_score, roc_curve, roc_auc_score, confusion_matrix\nimport random\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport optuna\nimport time\n\n# environment: {'kaggle','colab','jupyter'}\nenv = 'kaggle'\n\n# parameter tuning\nparam_tuning = False\n\n# random seed\nRANDOM_SEED = 17\n\n# models to run\nmodels = ['gb','hgb','xgb','ada']\n#models = ['xgb']\n\n# start time\nstart = time.time()","metadata":{"id":"_kIWQV22uWN2","execution":{"iopub.status.busy":"2022-04-05T19:34:27.797293Z","iopub.execute_input":"2022-04-05T19:34:27.797784Z","iopub.status.idle":"2022-04-05T19:34:27.803683Z","shell.execute_reply.started":"2022-04-05T19:34:27.797751Z","shell.execute_reply":"2022-04-05T19:34:27.802751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{"id":"-72YuRkGuWN4"}},{"cell_type":"code","source":"if env == 'kaggle':\n  path = '/kaggle/input/Kannada-MNIST/'\n\nelif env == 'colab':\n  from google.colab import drive\n  drive.mount('/content/drive')\n\n  path = '/content/drive/MyDrive/01_Tanulmanyok/ELTE AI&ML/02_Spring/Data Mining/Homeworks/Homework3/input/'\n\nelif env == 'jupyter':\n  path = './input/'\n\ntrain_data = pd.read_csv(path + 'train.csv')\ntest_data = pd.read_csv(path + 'test.csv')\n\nprint(train_data.shape)\ntrain_data.head()","metadata":{"executionInfo":{"elapsed":47109,"status":"ok","timestamp":1648625999600,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"},"user_tz":-120},"id":"Qv04GVJhuWN8","outputId":"3ba3ac93-16f7-448a-e316-516e0a31d9e3","execution":{"iopub.status.busy":"2022-04-05T19:34:27.835983Z","iopub.execute_input":"2022-04-05T19:34:27.836609Z","iopub.status.idle":"2022-04-05T19:34:32.626149Z","shell.execute_reply.started":"2022-04-05T19:34:27.836569Z","shell.execute_reply":"2022-04-05T19:34:32.625309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train-test split","metadata":{"id":"qzLJHjdSuWN9"}},{"cell_type":"code","source":"X, y = (train_data.drop(['label'], axis=1), train_data.label)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n\nX_train.shape","metadata":{"executionInfo":{"elapsed":1089,"status":"ok","timestamp":1648626027484,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"},"user_tz":-120},"id":"v6n45YDXuWOC","outputId":"d366537f-d7f6-4ccd-ba3f-28437f51ec8d","execution":{"iopub.status.busy":"2022-04-05T19:34:32.627473Z","iopub.execute_input":"2022-04-05T19:34:32.627702Z","iopub.status.idle":"2022-04-05T19:34:33.13984Z","shell.execute_reply.started":"2022-04-05T19:34:32.627674Z","shell.execute_reply":"2022-04-05T19:34:33.139029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Auxiliary functions","metadata":{"id":"IFKJIAnyuWOE"}},{"cell_type":"code","source":"# function to create plots about the model results\ndef get_figures(model, X_train, X_test, y_train, y_test):\n  y_pred = model.predict(X_test)\n  tr_pred = model.predict(X_train)\n\n  print(f'Train accuracy: {accuracy_score(y_train, tr_pred)}')\n  print(f'Val accuracy: {accuracy_score(y_test, y_pred)}')\n\n  y_prob = model.predict_proba(X_test)[:, 1]\n  fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2,2, figsize=(16,10))\n  plt.subplots_adjust(wspace=0.4, hspace=0.4)\n  fig.delaxes(ax4)\n\n  # confusion_matrix\n  C = confusion_matrix(y_test, y_pred)\n  sns.heatmap(C, ax=ax1, annot=True, fmt='g', cmap='icefire_r',\n  xticklabels=['False', 'True'], yticklabels=['False', 'True'])\n  ax1.set_ylabel('Actual')\n  ax1.set_xlabel('Predicted')\n  ax1.set_title('Confusion matrix')\n  plt.setp(ax1.get_yticklabels(), va=\"center\")\n\n  # ROC curve\n  fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n  sns.lineplot(fpr, tpr, ax=ax2, alpha=0.6, ci=None)\n  ax2.plot([0, 1], [0, 1], 'k--')\n  ax2.set_xlabel('False positive rate')\n  ax2.set_ylabel('True positive rate')\n  ax2.set_title('ROC curve')\n  ax2.legend(['Baseline', type(model).__name__])\n  num = round(roc_auc_score(y_test, y_prob)*100,2)\n  ax2.text(0.65, 0.25, 'AUC='+str(num)+'%', fontsize=15)","metadata":{"id":"IyDaaJkguWOF","execution":{"iopub.status.busy":"2022-04-05T19:34:33.141206Z","iopub.execute_input":"2022-04-05T19:34:33.142207Z","iopub.status.idle":"2022-04-05T19:34:33.152781Z","shell.execute_reply.started":"2022-04-05T19:34:33.142161Z","shell.execute_reply":"2022-04-05T19:34:33.151867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_accuracy_by_params(model, param, param_vals, n_sample=10000):\n  idxs = random.sample(list(X_train.index), n_sample)\n  grid = GridSearchCV(estimator=model, param_grid={param:param_vals}, cv=3, n_jobs=-1, scoring='accuracy')\n  grid.fit(X_train.loc[idxs], y_train.loc[idxs])\n  print(grid.cv_results_['mean_test_score'])\n  fig, ax = plt.subplots(1,1,figsize=(20,6))\n  ax = plt.plot(grid.cv_results_['mean_test_score'])\n  plt.xticks(range(len(param_vals)), param_vals)","metadata":{"id":"Y3bsS79yLzje","execution":{"iopub.status.busy":"2022-04-05T19:34:33.15441Z","iopub.execute_input":"2022-04-05T19:34:33.154715Z","iopub.status.idle":"2022-04-05T19:34:33.166085Z","shell.execute_reply.started":"2022-04-05T19:34:33.154686Z","shell.execute_reply":"2022-04-05T19:34:33.165297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling","metadata":{"id":"DbbfrhlKuWOG"}},{"cell_type":"markdown","source":"## Gradient Boosting\n\nhttps://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html","metadata":{"id":"3ooU9TlZuWOI"}},{"cell_type":"markdown","source":"### Initial hyperparameter tuning with GridSearchCV","metadata":{"id":"Ak1Rvbvm7Plu"}},{"cell_type":"code","source":"if param_tuning and 'gb' in models:\n  get_accuracy_by_params(GradientBoostingClassifier(), 'max_depth', [4,5,6,7,8], n_sample=1000)","metadata":{"executionInfo":{"elapsed":587054,"status":"ok","timestamp":1648562184393,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"},"user_tz":-120},"id":"SYviz-mRRp4I","outputId":"dd6336f2-d7b8-499e-e72a-b6ae79837176","execution":{"iopub.status.busy":"2022-04-05T19:34:33.167112Z","iopub.execute_input":"2022-04-05T19:34:33.167379Z","iopub.status.idle":"2022-04-05T19:34:33.180985Z","shell.execute_reply.started":"2022-04-05T19:34:33.16735Z","shell.execute_reply":"2022-04-05T19:34:33.18023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'gb' in models:\n  get_accuracy_by_params(GradientBoostingClassifier(), 'n_estimators', [100,150,200], n_sample=1000)","metadata":{"executionInfo":{"elapsed":288811,"status":"ok","timestamp":1648562487160,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"},"user_tz":-120},"id":"iQywsP58Ax0J","outputId":"5e891544-64e3-4345-99b0-235e5c834ead","execution":{"iopub.status.busy":"2022-04-05T19:34:33.182306Z","iopub.execute_input":"2022-04-05T19:34:33.182509Z","iopub.status.idle":"2022-04-05T19:34:33.191917Z","shell.execute_reply.started":"2022-04-05T19:34:33.182484Z","shell.execute_reply":"2022-04-05T19:34:33.191322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'gb' in models:\n  get_accuracy_by_params(GradientBoostingClassifier(), 'learning_rate', [0.05, 0.1, 0.5, 1], n_sample=1000)","metadata":{"executionInfo":{"elapsed":194506,"status":"ok","timestamp":1648640698678,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"},"user_tz":-120},"id":"_GzphM57A6Ol","outputId":"41e7a0f4-bb53-4c53-d044-eff43247e581","execution":{"iopub.status.busy":"2022-04-05T19:34:33.193097Z","iopub.execute_input":"2022-04-05T19:34:33.193296Z","iopub.status.idle":"2022-04-05T19:34:33.202654Z","shell.execute_reply.started":"2022-04-05T19:34:33.193272Z","shell.execute_reply":"2022-04-05T19:34:33.202105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final hyperarameter tuning with Optuna","metadata":{"id":"GnDtMqN-uWOJ"}},{"cell_type":"code","source":"def objective_gb(trial):\n  n_estimators = trial.suggest_int('n_estimators', 150, 200)\n  max_depth = trial.suggest_int('max_depth', 4, 5)\n  learning_rate = trial.suggest_float('learning_rate', 0.1, 0.5)\n\n  params = {\n      'n_estimators': n_estimators,\n      'max_depth': max_depth,\n      'learning_rate': learning_rate\n  }\n\n  gb = GradientBoostingClassifier(random_state=RANDOM_SEED)\n  gb.set_params(**params)\n\n  idxs = random.sample(list(X_train.index), 1000)\n  return -np.mean(cross_val_score(gb, X_train.loc[idxs], y_train.loc[idxs], cv=3))","metadata":{"id":"NpOQxLJ7fgCs","execution":{"iopub.status.busy":"2022-04-05T19:34:33.203562Z","iopub.execute_input":"2022-04-05T19:34:33.204164Z","iopub.status.idle":"2022-04-05T19:34:33.213436Z","shell.execute_reply.started":"2022-04-05T19:34:33.204131Z","shell.execute_reply":"2022-04-05T19:34:33.212809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'gb' in models:\n  gb_study = optuna.create_study()\n  gb_study.optimize(objective_gb, n_trials=10, timeout=2000)\n  params = gb_study.best_params\nelse:\n  params = {'learning_rate': 0.1333, 'max_depth': 5, 'n_estimators': 161} #0.984666\nprint(params)","metadata":{"id":"j6KtdEQWhIlC","executionInfo":{"status":"ok","timestamp":1648645237413,"user_tz":-120,"elapsed":638091,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"afc56a29-ad48-477f-f28e-ceb5b57feb6b","execution":{"iopub.status.busy":"2022-04-05T19:34:33.214293Z","iopub.execute_input":"2022-04-05T19:34:33.214957Z","iopub.status.idle":"2022-04-05T19:34:33.226422Z","shell.execute_reply.started":"2022-04-05T19:34:33.214924Z","shell.execute_reply":"2022-04-05T19:34:33.22552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'gb' in models:\n  gb = GradientBoostingClassifier()\n  gb.set_params(**params)\n  gb.fit(X_train, y_train)","metadata":{"id":"1KuY7YfX898t","executionInfo":{"status":"ok","timestamp":1648649332153,"user_tz":-120,"elapsed":4094774,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"44688ea8-af84-4841-b360-32a2f3865e13","execution":{"iopub.status.busy":"2022-04-05T19:34:33.228605Z","iopub.execute_input":"2022-04-05T19:34:33.229052Z","iopub.status.idle":"2022-04-05T20:37:56.762823Z","shell.execute_reply.started":"2022-04-05T19:34:33.228997Z","shell.execute_reply":"2022-04-05T20:37:56.76149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation","metadata":{"id":"A58oiBuguWOM"}},{"cell_type":"code","source":"if 'gb' in models:\n  y_pred_gb = gb.predict(X_test)\n  print(accuracy_score(y_test, y_pred_gb))","metadata":{"id":"7GAhM5nPuWON","executionInfo":{"status":"ok","timestamp":1648649333755,"user_tz":-120,"elapsed":1629,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"107b768f-3904-4afb-bb0e-a52f0d873e0b","execution":{"iopub.status.busy":"2022-04-05T20:37:56.765701Z","iopub.execute_input":"2022-04-05T20:37:56.766057Z","iopub.status.idle":"2022-04-05T20:37:58.080514Z","shell.execute_reply.started":"2022-04-05T20:37:56.765991Z","shell.execute_reply":"2022-04-05T20:37:58.07994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hist-Gradient Boosting\n\nhttps://scikit-learn.org/stable/modules/generated/sklearn.ensemble.HistGradientBoostingClassifier.html","metadata":{"id":"5SW9KsfhuWOP"}},{"cell_type":"markdown","source":"### Initial hyperparameter tuning with GridSearchCV","metadata":{"id":"NhE2URZmuWOQ"}},{"cell_type":"code","source":"if param_tuning and 'hgb' in models:\n  get_accuracy_by_params(HistGradientBoostingClassifier(), 'learning_rate', [0.05, 0.1, 0.5, 1], n_sample=10000)","metadata":{"id":"PICnGBjLuWOR","executionInfo":{"status":"ok","timestamp":1648653879403,"user_tz":-120,"elapsed":742473,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"cc1a1dda-ca63-4753-e65e-13acecd5668a","execution":{"iopub.status.busy":"2022-04-05T20:37:58.081479Z","iopub.execute_input":"2022-04-05T20:37:58.082002Z","iopub.status.idle":"2022-04-05T20:37:58.085869Z","shell.execute_reply.started":"2022-04-05T20:37:58.08197Z","shell.execute_reply":"2022-04-05T20:37:58.085297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'hgb' in models:\n  get_accuracy_by_params(HistGradientBoostingClassifier(learning_rate = 0.1), 'max_depth', [5,7,10,15,25,50], n_sample=10000)","metadata":{"id":"8Ox9_47sTgkp","executionInfo":{"status":"ok","timestamp":1648655806407,"user_tz":-120,"elapsed":1461599,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"7bfd7bc9-3914-458c-d2ed-d151c7ea2a09","execution":{"iopub.status.busy":"2022-04-05T20:37:58.086822Z","iopub.execute_input":"2022-04-05T20:37:58.087541Z","iopub.status.idle":"2022-04-05T20:37:58.102392Z","shell.execute_reply.started":"2022-04-05T20:37:58.0875Z","shell.execute_reply":"2022-04-05T20:37:58.1014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'hgb' in models:\n  get_accuracy_by_params(HistGradientBoostingClassifier(learning_rate=0.1, max_depth=10), 'max_leaf_nodes', [30, 50, 75, 100], n_sample=10000)","metadata":{"id":"uGghRwiaTgYj","executionInfo":{"status":"ok","timestamp":1648662034717,"user_tz":-120,"elapsed":1513076,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"1a824ddd-60c0-4595-87f3-37afa5785415","execution":{"iopub.status.busy":"2022-04-05T20:37:58.103526Z","iopub.execute_input":"2022-04-05T20:37:58.103732Z","iopub.status.idle":"2022-04-05T20:37:58.1136Z","shell.execute_reply.started":"2022-04-05T20:37:58.103708Z","shell.execute_reply":"2022-04-05T20:37:58.112848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final hyperarameter tuning with Optuna","metadata":{"id":"LLM7VolUKs2o"}},{"cell_type":"code","source":"def objective_hgb(trial):\n  n_estimators = trial.suggest_int('max_leaf_nodes', 70, 80)\n  max_depth = trial.suggest_int('max_depth', 8, 12)\n  learning_rate = trial.suggest_float('learning_rate', 0.05, 0.15)\n\n  params = {\n      'max_leaf_nodes': n_estimators,\n      'max_depth': max_depth,\n      'learning_rate': learning_rate\n  }\n\n  hgb = HistGradientBoostingClassifier(random_state=RANDOM_SEED)\n  hgb.set_params(**params)\n\n  idxs = random.sample(list(X_train.index), 1000)\n  return -np.mean(cross_val_score(hgb, X_train.loc[idxs], y_train.loc[idxs], cv=3))","metadata":{"id":"4BcDFiz0KuMT","execution":{"iopub.status.busy":"2022-04-05T20:37:58.115101Z","iopub.execute_input":"2022-04-05T20:37:58.115562Z","iopub.status.idle":"2022-04-05T20:37:58.125432Z","shell.execute_reply.started":"2022-04-05T20:37:58.115523Z","shell.execute_reply":"2022-04-05T20:37:58.124795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'hgb' in models:\n  hgb_study = optuna.create_study()\n  hgb_study.optimize(objective_hgb, n_trials=10, timeout=2000)\n  params = hgb_study.best_params\nelse:\n  params = {'learning_rate': 0.136, 'max_depth': 9, 'max_leaf_nodes': 78} #0.9846666\nprint(params)","metadata":{"id":"WHYJjr8Naxew","executionInfo":{"status":"ok","timestamp":1648662860565,"user_tz":-120,"elapsed":707688,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"3bcb918d-34dd-49d1-c80d-429f16abc113","execution":{"iopub.status.busy":"2022-04-05T20:37:58.126898Z","iopub.execute_input":"2022-04-05T20:37:58.127461Z","iopub.status.idle":"2022-04-05T20:37:58.137207Z","shell.execute_reply.started":"2022-04-05T20:37:58.12742Z","shell.execute_reply":"2022-04-05T20:37:58.136362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'hgb' in models:\n  hgb = HistGradientBoostingClassifier()\n  hgb.set_params(**params)\n  hgb.fit(X_train, y_train)","metadata":{"id":"y7gt6QwXaxNy","executionInfo":{"status":"ok","timestamp":1648663216102,"user_tz":-120,"elapsed":287228,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"56fb6a73-e38c-49b8-80fb-62bd427fef21","execution":{"iopub.status.busy":"2022-04-05T20:37:58.138469Z","iopub.execute_input":"2022-04-05T20:37:58.138721Z","iopub.status.idle":"2022-04-05T20:40:22.869151Z","shell.execute_reply.started":"2022-04-05T20:37:58.138695Z","shell.execute_reply":"2022-04-05T20:40:22.868218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation","metadata":{"id":"pCsqZ_pZuWOS"}},{"cell_type":"code","source":"if 'hgb' in models:\n  y_pred_hgb = gb.predict(X_test)\n  print(accuracy_score(y_test, y_pred_hgb))","metadata":{"id":"h7KPtPWQuWOS","executionInfo":{"status":"ok","timestamp":1648663217921,"user_tz":-120,"elapsed":1844,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"d506adf3-d312-4d8c-c0db-02d255f06381","execution":{"iopub.status.busy":"2022-04-05T20:40:22.870565Z","iopub.execute_input":"2022-04-05T20:40:22.871168Z","iopub.status.idle":"2022-04-05T20:40:24.144905Z","shell.execute_reply.started":"2022-04-05T20:40:22.871124Z","shell.execute_reply":"2022-04-05T20:40:24.144109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## XGB\n\nhttps://xgboost.readthedocs.io/en/stable/python/python_api.html","metadata":{"id":"3UnUEl2tuWOT"}},{"cell_type":"markdown","source":"### Initial hyperparameter tuning with GridSearchCV","metadata":{"id":"E513jFMjuWOT"}},{"cell_type":"code","source":"if param_tuning and 'xgb' in models:\n  get_accuracy_by_params(XGBClassifier(), 'max_depth', [4,5,6,7,8], n_sample=1000)","metadata":{"id":"ZEKzMkqCJ1gW","executionInfo":{"status":"ok","timestamp":1648568511239,"user_tz":-120,"elapsed":239764,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"6ce8b5b2-cfc0-4dba-ab3e-f53965875899","execution":{"iopub.status.busy":"2022-04-05T20:40:24.146087Z","iopub.execute_input":"2022-04-05T20:40:24.146472Z","iopub.status.idle":"2022-04-05T20:40:24.150558Z","shell.execute_reply.started":"2022-04-05T20:40:24.146442Z","shell.execute_reply":"2022-04-05T20:40:24.149768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'xgb' in models:\n  get_accuracy_by_params(XGBClassifier(max_depth=6), 'n_estimators', [50,100,150,200], n_sample=1000)","metadata":{"id":"cODS50-3KGy2","executionInfo":{"status":"ok","timestamp":1648665381773,"user_tz":-120,"elapsed":285545,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"5825248f-71f7-4446-bb26-28ae6fe9cdab","execution":{"iopub.status.busy":"2022-04-05T20:40:24.151778Z","iopub.execute_input":"2022-04-05T20:40:24.152283Z","iopub.status.idle":"2022-04-05T20:40:24.16338Z","shell.execute_reply.started":"2022-04-05T20:40:24.152243Z","shell.execute_reply":"2022-04-05T20:40:24.162547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'xgb' in models:\n  get_accuracy_by_params(XGBClassifier(), 'learning_rate', [0.05, 0.1, 0.5, 1], n_sample=1000)","metadata":{"id":"xtshgk_vKOBp","executionInfo":{"status":"ok","timestamp":1648640813173,"user_tz":-120,"elapsed":114540,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"04d575f2-48c8-4e32-df9b-60ab168fc2ec","execution":{"iopub.status.busy":"2022-04-05T20:40:24.16454Z","iopub.execute_input":"2022-04-05T20:40:24.164822Z","iopub.status.idle":"2022-04-05T20:40:24.174967Z","shell.execute_reply.started":"2022-04-05T20:40:24.164786Z","shell.execute_reply":"2022-04-05T20:40:24.174375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'xgb' in models:\n  get_accuracy_by_params(XGBClassifier(), 'colsample_bytree', [0.1, 0.2, 0.3, 0.4], n_sample=1000)","metadata":{"id":"FFR3X7DGKVeg","executionInfo":{"status":"ok","timestamp":1648568854109,"user_tz":-120,"elapsed":45866,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"f44847d4-3485-4492-936f-fec27d96e53c","execution":{"iopub.status.busy":"2022-04-05T20:40:24.175863Z","iopub.execute_input":"2022-04-05T20:40:24.176613Z","iopub.status.idle":"2022-04-05T20:40:24.187548Z","shell.execute_reply.started":"2022-04-05T20:40:24.176565Z","shell.execute_reply":"2022-04-05T20:40:24.186636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'xgb' in models:\n  get_accuracy_by_params(XGBClassifier(), 'subsample', [0.2, 0.3, 0.4, 0.5], n_sample=1000)","metadata":{"id":"ifu0m1RdKYe9","executionInfo":{"status":"ok","timestamp":1648568979422,"user_tz":-120,"elapsed":125342,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"f7aee9ed-3a7a-4249-de38-59d19bd0000e","execution":{"iopub.status.busy":"2022-04-05T20:40:24.189946Z","iopub.execute_input":"2022-04-05T20:40:24.190216Z","iopub.status.idle":"2022-04-05T20:40:24.203757Z","shell.execute_reply.started":"2022-04-05T20:40:24.190187Z","shell.execute_reply":"2022-04-05T20:40:24.202896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final hyperarameter tuning with Optuna","metadata":{"id":"mFcFmbLFKoJT"}},{"cell_type":"code","source":"def objective_xgb(trial):\n  n_estimators = trial.suggest_int('n_estimators', 140, 180)\n  max_depth = trial.suggest_int('max_depth', 5, 6)\n  learning_rate = trial.suggest_float('learning_rate', 0.3, 0.7)\n  colsample_bytree = trial.suggest_float('colsample_bytree', 0.05, 0.15)\n  subsample = trial.suggest_float('subsample', 0.5, 0.6)\n\n  params = {\n      'n_estimators': n_estimators,\n      'max_depth': max_depth,\n      'learning_rate': learning_rate,\n      'colsample_bytree': colsample_bytree,\n      'subsample': subsample\n  }\n\n  xgb = XGBClassifier(random_state=RANDOM_SEED)\n  xgb.set_params(**params)\n\n  idxs = random.sample(list(X_train.index), 1000)\n  return -np.mean(cross_val_score(xgb, X_train.loc[idxs], y_train.loc[idxs], cv=3))","metadata":{"id":"WLHKuq5PKnmd","execution":{"iopub.status.busy":"2022-04-05T20:40:24.206314Z","iopub.execute_input":"2022-04-05T20:40:24.206555Z","iopub.status.idle":"2022-04-05T20:40:24.21509Z","shell.execute_reply.started":"2022-04-05T20:40:24.206526Z","shell.execute_reply":"2022-04-05T20:40:24.214517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'xgb' in models:\n  xgb_study = optuna.create_study()\n  xgb_study.optimize(objective_gb, n_trials=10, timeout=2000)\n  params = xgb_study.best_params\nelse:\n  params = {'learning_rate': 0.1784, 'max_depth': 4, 'n_estimators': 184} #0.98475\nprint(params)","metadata":{"id":"It2sWd10LD9A","executionInfo":{"status":"ok","timestamp":1648650440328,"user_tz":-120,"elapsed":504159,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"6df3977a-1e8e-4bc3-c9db-cbbee375794f","execution":{"iopub.status.busy":"2022-04-05T20:40:24.215976Z","iopub.execute_input":"2022-04-05T20:40:24.218199Z","iopub.status.idle":"2022-04-05T20:40:24.225861Z","shell.execute_reply.started":"2022-04-05T20:40:24.218155Z","shell.execute_reply":"2022-04-05T20:40:24.225105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'xgb' in models:\n  xgb = XGBClassifier()\n  xgb.set_params(**params)\n  xgb.fit(X_train, y_train)","metadata":{"id":"66f4Z5TVLFYO","executionInfo":{"status":"ok","timestamp":1648652192749,"user_tz":-120,"elapsed":1752437,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"aa623217-9d3b-49ec-a5a0-b42c8d06bf94","execution":{"iopub.status.busy":"2022-04-05T20:40:24.227065Z","iopub.execute_input":"2022-04-05T20:40:24.227681Z","iopub.status.idle":"2022-04-05T20:54:17.855171Z","shell.execute_reply.started":"2022-04-05T20:40:24.227639Z","shell.execute_reply":"2022-04-05T20:54:17.854325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation","metadata":{"id":"VaYWlcg1uWOV"}},{"cell_type":"code","source":"if 'xgb' in models:\n  y_pred_xgb = xgb.predict(X_test)\n  print(accuracy_score(y_test, y_pred_xgb))","metadata":{"id":"l11Dye9UuWOW","executionInfo":{"status":"ok","timestamp":1648652196283,"user_tz":-120,"elapsed":2272,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"3142c4e7-1293-45a4-bdb4-03fbb7b9b67f","execution":{"iopub.status.busy":"2022-04-05T20:54:17.856785Z","iopub.execute_input":"2022-04-05T20:54:17.85712Z","iopub.status.idle":"2022-04-05T20:54:18.149423Z","shell.execute_reply.started":"2022-04-05T20:54:17.857078Z","shell.execute_reply":"2022-04-05T20:54:18.148573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## AdaBoost\n\nhttps://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html","metadata":{"id":"WQYWOGAouWOW"}},{"cell_type":"markdown","source":"### Initial hyperparameter tuning with GridSearchCV","metadata":{"id":"MslDoJ1BuWOW"}},{"cell_type":"code","source":"if param_tuning and 'ada' in models:\n  get_accuracy_by_params(AdaBoostClassifier(), 'learning_rate', [0.001, 0.05, 0.1, 0.5, 1], n_sample=1000)","metadata":{"id":"Bcmgyof1uWOX","executionInfo":{"status":"ok","timestamp":1648652303468,"user_tz":-120,"elapsed":13883,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"2dad73d1-ac9c-45f5-8acd-775b4362f4b9","execution":{"iopub.status.busy":"2022-04-05T20:54:18.153873Z","iopub.execute_input":"2022-04-05T20:54:18.15418Z","iopub.status.idle":"2022-04-05T20:54:18.160109Z","shell.execute_reply.started":"2022-04-05T20:54:18.154147Z","shell.execute_reply":"2022-04-05T20:54:18.159447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'ada' in models:\n  get_accuracy_by_params(AdaBoostClassifier(), 'n_estimators', [10,50,100,150,200,250], n_sample=1000)","metadata":{"id":"pzMEaKWOeOM_","executionInfo":{"status":"ok","timestamp":1648640118063,"user_tz":-120,"elapsed":34561,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"8fb97787-9063-4eab-c368-8c1f6d9b43a8","execution":{"iopub.status.busy":"2022-04-05T20:54:18.161143Z","iopub.execute_input":"2022-04-05T20:54:18.161478Z","iopub.status.idle":"2022-04-05T20:54:18.172429Z","shell.execute_reply.started":"2022-04-05T20:54:18.161436Z","shell.execute_reply":"2022-04-05T20:54:18.171674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'ada' in models:\n  get_accuracy_by_params(AdaBoostClassifier(), 'algorithm', ['SAMME', 'SAMME.R'], n_sample=1000)","metadata":{"id":"xGZSDw3NeN0l","executionInfo":{"status":"ok","timestamp":1648638156077,"user_tz":-120,"elapsed":6865,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"bb401287-94cf-42ce-b339-a94b843634dd","execution":{"iopub.status.busy":"2022-04-05T20:54:18.174722Z","iopub.execute_input":"2022-04-05T20:54:18.175311Z","iopub.status.idle":"2022-04-05T20:54:18.182264Z","shell.execute_reply.started":"2022-04-05T20:54:18.175271Z","shell.execute_reply":"2022-04-05T20:54:18.181552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final hyperarameter tuning with Optuna","metadata":{"id":"h4e2O857eK-3"}},{"cell_type":"code","source":"def objective_ada(trial):\n  n_estimators = trial.suggest_int('n_estimators', 230, 270)\n  algorithm = trial.suggest_categorical('algorithm', ['SAMME'])\n  learning_rate = trial.suggest_float('learning_rate', 0.03, 0.07)\n  \n  params = {\n      'n_estimators': n_estimators,\n      'algorithm': algorithm,\n      'learning_rate': learning_rate\n  }\n\n  ada = AdaBoostClassifier(random_state=RANDOM_SEED)\n  ada.set_params(**params)\n\n  idxs = random.sample(list(X_train.index), 1000)\n  return -np.mean(cross_val_score(ada, X_train.loc[idxs], y_train.loc[idxs], cv=3))","metadata":{"id":"368V8TW1eL8I","execution":{"iopub.status.busy":"2022-04-05T20:54:18.183682Z","iopub.execute_input":"2022-04-05T20:54:18.183958Z","iopub.status.idle":"2022-04-05T20:54:18.193532Z","shell.execute_reply.started":"2022-04-05T20:54:18.183921Z","shell.execute_reply":"2022-04-05T20:54:18.192873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if param_tuning and 'ada' in models:\n  ada_study = optuna.create_study()\n  ada_study.optimize(objective_ada, n_trials=10, timeout=2000)\n  params = ada_study.best_params\nelse:\n  params = {'learning_rate': 0.063588, 'n_estimators': 232, 'algorithm': 'SAMME'} #0.54658\nprint(params)","metadata":{"id":"FYIaNREhhmFE","executionInfo":{"status":"ok","timestamp":1648652453290,"user_tz":-120,"elapsed":102404,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"cb3c2598-495f-46d1-bab8-a2886435c134","execution":{"iopub.status.busy":"2022-04-05T20:54:18.19457Z","iopub.execute_input":"2022-04-05T20:54:18.195198Z","iopub.status.idle":"2022-04-05T20:54:18.208425Z","shell.execute_reply.started":"2022-04-05T20:54:18.19516Z","shell.execute_reply":"2022-04-05T20:54:18.20759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'ada' in models:\n  ada = AdaBoostClassifier()\n  ada.set_params(**params)\n  ada.fit(X_train, y_train)","metadata":{"id":"MfLXtf3ahsd9","executionInfo":{"status":"ok","timestamp":1648652652404,"user_tz":-120,"elapsed":160255,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"09a4f665-2bb3-47be-cd4e-eaeed3d3db6e","execution":{"iopub.status.busy":"2022-04-05T20:54:18.20951Z","iopub.execute_input":"2022-04-05T20:54:18.210032Z","iopub.status.idle":"2022-04-05T20:56:52.020996Z","shell.execute_reply.started":"2022-04-05T20:54:18.209996Z","shell.execute_reply":"2022-04-05T20:56:52.019819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation","metadata":{"id":"uH-zhMVRuWOX"}},{"cell_type":"code","source":"if 'ada' in models:\n  y_pred_ada = ada.predict(X_test)\n  print(accuracy_score(y_test, y_pred_ada))","metadata":{"id":"NNEKt0FvuWOY","executionInfo":{"status":"ok","timestamp":1648653060768,"user_tz":-120,"elapsed":6344,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"1a344568-d085-4ac2-b82b-fe87e1fdcef3","execution":{"iopub.status.busy":"2022-04-05T20:56:52.022534Z","iopub.execute_input":"2022-04-05T20:56:52.022769Z","iopub.status.idle":"2022-04-05T20:56:56.740324Z","shell.execute_reply.started":"2022-04-05T20:56:52.02274Z","shell.execute_reply":"2022-04-05T20:56:56.739338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Eval of mixed prediction","metadata":{}},{"cell_type":"code","source":"def get_most_common_vals(list_of_values):\n  return [max(set(p), key = list(p).count) for p in np.vstack(list_of_values).T]","metadata":{"execution":{"iopub.status.busy":"2022-04-05T20:56:56.741658Z","iopub.execute_input":"2022-04-05T20:56:56.741866Z","iopub.status.idle":"2022-04-05T20:56:56.747862Z","shell.execute_reply.started":"2022-04-05T20:56:56.74184Z","shell.execute_reply":"2022-04-05T20:56:56.747108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = get_most_common_vals([y_pred_xgb, y_pred_gb, y_pred_hgb])\nprint(accuracy_score(y_test, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-04-05T20:56:56.749619Z","iopub.execute_input":"2022-04-05T20:56:56.749924Z","iopub.status.idle":"2022-04-05T20:56:56.798653Z","shell.execute_reply.started":"2022-04-05T20:56:56.749885Z","shell.execute_reply":"2022-04-05T20:56:56.797902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"id":"RRoyL9lJuWOe"}},{"cell_type":"code","source":"ids, test_set = test_data.id, test_data.drop(['id'], axis=1)\nxgb_preds = xgb.predict(test_set)\ngb_preds = gb.predict(test_set)\nhgb_preds = hgb.predict(test_set)\n\nfinal_preds = get_most_common_vals([xgb_preds, gb_preds, hgb_preds])","metadata":{"id":"wN1fujuQuWOe","execution":{"iopub.status.busy":"2022-04-05T20:56:56.799689Z","iopub.execute_input":"2022-04-05T20:56:56.800002Z","iopub.status.idle":"2022-04-05T20:56:57.628746Z","shell.execute_reply.started":"2022-04-05T20:56:56.799974Z","shell.execute_reply":"2022-04-05T20:56:57.628126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(final_preds, index=ids, name='label').to_csv('/kaggle/working/submission.csv')","metadata":{"id":"Jr0V4jtTuWOg","executionInfo":{"status":"error","timestamp":1648571875664,"user_tz":-120,"elapsed":31,"user":{"displayName":"Bence Kátai-Pál","userId":"09798098129297838721"}},"outputId":"261a9e45-7377-410e-a9d0-04981b331364","execution":{"iopub.status.busy":"2022-04-05T20:56:57.629867Z","iopub.execute_input":"2022-04-05T20:56:57.630527Z","iopub.status.idle":"2022-04-05T20:56:57.652449Z","shell.execute_reply.started":"2022-04-05T20:56:57.63049Z","shell.execute_reply":"2022-04-05T20:56:57.651609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end = time.time()\nprint(f'Time of execution:', time.strftime('%H:%M:%S',time.gmtime(end-start)))","metadata":{"execution":{"iopub.status.busy":"2022-04-05T20:56:57.653837Z","iopub.execute_input":"2022-04-05T20:56:57.654187Z","iopub.status.idle":"2022-04-05T20:56:57.659306Z","shell.execute_reply.started":"2022-04-05T20:56:57.654158Z","shell.execute_reply":"2022-04-05T20:56:57.658106Z"},"trusted":true},"execution_count":null,"outputs":[]}]}