{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67223,"databundleVersionId":7463838,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benchmarking GBDT for tabular data\n\nTabular data benchmark: ensemble of 5 HistGradient boosting models with early stopping. Each model in the ensemble uses a different training-validation split for early stopping. Note that we use slightly different hyperparameters from the defaults.","metadata":{}},{"cell_type":"code","source":"%%capture\n# Installing v1.4.1 - histgradient boosting models can infer categorical variabkes\n# from pandas dataframes\n!pip install \"scikit-learn>=1.4\"","metadata":{"execution":{"iopub.status.busy":"2024-03-26T20:02:05.914390Z","iopub.execute_input":"2024-03-26T20:02:05.915391Z","iopub.status.idle":"2024-03-26T20:02:26.641645Z","shell.execute_reply.started":"2024-03-26T20:02:05.915349Z","shell.execute_reply":"2024-03-26T20:02:26.640280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nimport os\nimport json\nfrom sklearn.metrics import log_loss, mean_squared_error\nfrom sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-26T20:03:44.146759Z","iopub.execute_input":"2024-03-26T20:03:44.147797Z","iopub.status.idle":"2024-03-26T20:03:44.518386Z","shell.execute_reply.started":"2024-03-26T20:03:44.147758Z","shell.execute_reply":"2024-03-26T20:03:44.517170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'max_iter': 1000,\n    'learning_rate': 0.05,\n    'categorical_features': 'from_dtype',\n    'early_stopping': True,\n    'scoring': 'loss',\n    'validation_fraction': 0.1,\n    'n_iter_no_change': 20\n}","metadata":{"execution":{"iopub.status.busy":"2024-03-26T20:04:37.005056Z","iopub.execute_input":"2024-03-26T20:04:37.006235Z","iopub.status.idle":"2024-03-26T20:04:37.012503Z","shell.execute_reply.started":"2024-03-26T20:04:37.006184Z","shell.execute_reply":"2024-03-26T20:04:37.011296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/bench-tab-v1/'\n\ntasks = json.load(open(f'{path}train/dict_task.json'))\nsuits = json.load(open(f'{path}train/dict_suit.json'))\n\nDEBUG = False\nsuits = {'334': [361110, 361111], '335': [361093]} if DEBUG else suits\nn_seeds = 2 if DEBUG else 5\n\ndef prepare_sub(data, identifier):\n    data = pd.DataFrame(data)\n    data.columns = ['target']\n    data['ID'] =  identifier + '_' + data.index.astype(str)\n    columns = ['ID','target']\n    return data[columns]\n\nres_gbdt = {}\nn_trees = {}\nsub = []\n\nfor t in suits:\n    print(t)\n    for suit in suits[t]:\n        \n        identifier = f'{t}_{suit}'\n        task_type = tasks[t]\n        var_type = task_type['features']\n        target_type = task_type['target']\n\n        X = pd.read_parquet(f'{path}train/{identifier}/data_train.parquet')\n        X_test = pd.read_parquet(f'{path}train/{identifier}/data_test.parquet')\n        y = pd.read_parquet(f'{path}train/{identifier}/target_train.parquet').values.ravel()\n        \n        names = json.load(open(f'{path}train/{identifier}/attribute_names.json'))\n        cat_indicator = json.load(open(f'{path}train/{identifier}/categorical_indicator.json'))\n        \n        preds = []\n        n_trees[identifier] = []\n        \n        for seed in range(n_seeds):\n            if target_type == 'num':\n                gbdt_model = HistGradientBoostingRegressor(random_state=seed, **params)\n            else:\n                gbdt_model = HistGradientBoostingClassifier(random_state=seed, **params)\n                \n            gbdt_model = gbdt_model.fit(X, y)\n            # record the number of treees after early stopping\n            n_trees[identifier].append(gbdt_model.n_iter_)\n            \n            if target_type=='num':\n                preds.append(gbdt_model.predict(X_test))\n            else:\n                preds.append(gbdt_model.predict_proba(X_test)[:,1])\n            \n        preds = np.mean(np.array(preds),axis=0)\n\n        preds = pd.DataFrame(preds)\n        preds.index = X_test.index\n        \n        # appends empty tests / solutions\n        sub.append(prepare_sub(preds, identifier))\n        \n        print('')\n        print(f'{identifier=} {var_type=} {target_type=}')","metadata":{"execution":{"iopub.status.busy":"2024-03-26T20:09:52.322780Z","iopub.execute_input":"2024-03-26T20:09:52.323209Z","iopub.status.idle":"2024-03-26T20:10:07.890095Z","shell.execute_reply.started":"2024-03-26T20:09:52.323175Z","shell.execute_reply":"2024-03-26T20:10:07.888858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.concat(sub).reset_index(drop=True)\ndisplay(sub.head())\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T20:06:54.159161Z","iopub.execute_input":"2024-03-26T20:06:54.159491Z","iopub.status.idle":"2024-03-26T20:06:54.275927Z","shell.execute_reply.started":"2024-03-26T20:06:54.159463Z","shell.execute_reply":"2024-03-26T20:06:54.274769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save number of trees\nimport json\nwith open('final_no_of_trees.json', 'w') as f:\n    json.dump(n_trees, f)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T20:11:51.598829Z","iopub.execute_input":"2024-03-26T20:11:51.599823Z","iopub.status.idle":"2024-03-26T20:11:51.605346Z","shell.execute_reply.started":"2024-03-26T20:11:51.599781Z","shell.execute_reply":"2024-03-26T20:11:51.604417Z"},"trusted":true},"execution_count":null,"outputs":[]}]}