{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67223,"databundleVersionId":7463838,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benchmarking GBDT for tabular data\n\nTabular data benchmark: Xgboost model with hyperparameters tuned sepearately for regression and classification tasks. For regression tasks, the hyperparameters were tuned to minimize the average normalized root mean squared error across all the tasks. For classification tasks, they were tuned to minimized average normalized cross-entropy loss (https://research.facebook.com/publications/practical-lessons-from-predicting-clicks-on-ads-at-facebook/) for classification tasks. These hyperparameters are slightly better overall than the default hyperparameters. The hyperparameter tuning was performed using the multi-task Bayesian optimization framework (https://papers.nips.cc/paper_files/paper/2013/file/f33ba15effa5c10e873bf3842afb46a6-Paper.pdf). ","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nimport os\nimport json\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom sklearn.base import clone","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-22T19:38:08.967021Z","iopub.execute_input":"2024-03-22T19:38:08.967488Z","iopub.status.idle":"2024-03-22T19:38:08.974872Z","shell.execute_reply.started":"2024-03-22T19:38:08.967451Z","shell.execute_reply":"2024-03-22T19:38:08.972924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_hypers = {\n    'num': {\n        'colsample_bytree': 0.6724257944373457,\n        'gamma': 2.2591336004191074e-08,\n        'learning_rate': 0.04188773746654915,\n        'max_depth': 5,\n        'min_child_weight': 1.0,\n        'n_estimators': 703,\n        'reg_alpha': 1e-08,\n        'reg_lambda': 0.04624204609161937,\n        'subsample': 0.5966345758883806\n    },\n    'cat': {\n        'colsample_bytree': 1.0,\n        'gamma': 1.3882311710764983e-06,\n        'learning_rate': 0.03213345136739538,\n        'max_depth': 7,\n        'min_child_weight': 2.2201243841928835,\n        'n_estimators': 859,\n        'reg_alpha': 8.849752017536362,\n        'reg_lambda': 1.1384936550227305e-08,\n        'subsample': 0.7974752061543793\n    }\n}\n","metadata":{"execution":{"iopub.status.busy":"2024-03-22T19:38:16.097817Z","iopub.execute_input":"2024-03-22T19:38:16.098403Z","iopub.status.idle":"2024-03-22T19:38:16.106573Z","shell.execute_reply.started":"2024-03-22T19:38:16.098356Z","shell.execute_reply":"2024-03-22T19:38:16.105547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/bench-tab-v1/'\n\ntasks = json.load(open(f'{path}train/dict_task.json'))\nsuits = json.load(open(f'{path}train/dict_suit.json'))\n\n\nDEBUG = False\nsuits = {'334': [361110, 361111], '335': [361093]} if DEBUG else suits\n\n\ndef prepare_sub(data, identifier):\n    data = pd.DataFrame(data)\n    data.columns = ['target']\n    data['ID'] =  identifier + '_' + data.index.astype(str)\n    columns = ['ID','target']\n    return data[columns]\n\nres_gbdt = {}\nsub = []\n\nfor t in suits:\n    print(t)\n    for suit in suits[t]:\n        \n        identifier = f'{t}_{suit}'\n        task_type = tasks[t]\n        var_type = task_type['features']\n        target_type = task_type['target']\n\n        X = pd.read_parquet(f'{path}train/{identifier}/data_train.parquet')\n        X_test = pd.read_parquet(f'{path}train/{identifier}/data_test.parquet')\n        y = pd.read_parquet(f'{path}train/{identifier}/target_train.parquet')\n        \n        names = json.load(open(f'{path}train/{identifier}/attribute_names.json'))\n        cat_indicator = json.load(open(f'{path}train/{identifier}/categorical_indicator.json'))\n        \n            \n        est_class = XGBRegressor if target_type == 'num' else XGBClassifier\n\n        gbdt_model = est_class(\n            tree_method= 'hist', n_jobs = -1, \n            enable_categorical = any(cat_indicator),\n            **xgb_hypers[target_type]\n        )\n\n        gbdt_model = gbdt_model.fit(X, y.values)\n            \n        if target_type=='num':\n            preds = gbdt_model.predict(X_test)\n        else:\n            preds = gbdt_model.predict_proba(X_test)[:,1]\n            \n\n        preds = pd.DataFrame(preds)\n        preds.index = X_test.index\n        \n        # appends empty tests / solutions\n        sub.append(prepare_sub(preds, identifier))\n        \n        print('')\n        print(f'{identifier=} {var_type=} {target_type=}')","metadata":{"execution":{"iopub.status.busy":"2024-03-22T19:43:13.290414Z","iopub.execute_input":"2024-03-22T19:43:13.290941Z","iopub.status.idle":"2024-03-22T19:43:22.068490Z","shell.execute_reply.started":"2024-03-22T19:43:13.290903Z","shell.execute_reply":"2024-03-22T19:43:22.066338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.concat(sub).reset_index(drop=True)\ndisplay(sub.head())\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-22T19:47:58.671373Z","iopub.execute_input":"2024-03-22T19:47:58.672008Z","iopub.status.idle":"2024-03-22T19:47:58.786791Z","shell.execute_reply.started":"2024-03-22T19:47:58.671956Z","shell.execute_reply":"2024-03-22T19:47:58.785278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}