{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67223,"databundleVersionId":7463838,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benchmarking GBDT for tabular data\n\nTabular data benchmark: LightGBM models with hyperparameters tuned through optuna. ","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nimport os\nimport re\nimport json\nfrom sklearn.metrics import log_loss, mean_squared_error\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom lightgbm import LGBMClassifier, LGBMRegressor\nfrom typing import Optional, Dict\nfrom functools import partial\n\nimport optuna\nimport joblib","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-09T22:13:17.139687Z","iopub.execute_input":"2024-05-09T22:13:17.140106Z","iopub.status.idle":"2024-05-09T22:13:17.146623Z","shell.execute_reply.started":"2024-05-09T22:13:17.140071Z","shell.execute_reply":"2024-05-09T22:13:17.145383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nSAVE_DIR = 'optuna_trials'\n\nif DEBUG:\n    MAX_TRIALS = 15\n    MAX_RUNTIME = 150 # 2.5 minutes\nelse:\n    MAX_TRIALS = 50\n    MAX_RUNTIME = 450 # 7.5 minutes\n    # disable optuna logging info\n    optuna.logging.set_verbosity(optuna.logging.WARNING)\n    \n# CREATE save directory for optuna trials\nos.makedirs(SAVE_DIR)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T22:13:18.601928Z","iopub.execute_input":"2024-05-09T22:13:18.602403Z","iopub.status.idle":"2024-05-09T22:13:18.718305Z","shell.execute_reply.started":"2024-05-09T22:13:18.602355Z","shell.execute_reply":"2024-05-09T22:13:18.716901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_and_test_fold(\n    target_type: str,\n    X: pd.DataFrame, y: np.ndarray,\n    train_index: np.ndarray, test_index: np.ndarray,\n    params:Dict= {}\n) -> float:\n    \n    X_train, X_test = X.iloc[train_index,:], X.iloc[test_index, :]\n    y_train, y_test = y[train_index], y[test_index]\n    \n    if target_type == 'num':\n        model = LGBMRegressor(n_jobs=4, verbose=-1, **params)\n    else:\n        model = LGBMClassifier(n_jobs=4, verbose=-1, **params)\n    \n    # fit model on training data\n    _ = model.fit(X_train, y_train)\n    \n    # generate predictions and compute score\n    if target_type == 'num':\n        y_test_pred = model.predict(X_test)\n        return mean_squared_error(y_test, y_test_pred)\n        \n    y_test_pred = model.predict_proba(X_test)[:, -1]\n    return log_loss(y_test, y_test_pred)\n\n\ndef optuna_objective(trial: optuna.Trial, X:pd.DataFrame, y:np.ndarray, target_type:str):\n    params = {\n        'n_estimators':trial.suggest_int('n_estimators',50, 1000,log=True),\n        'learning_rate': trial.suggest_float('learning_rate', 5e-3, 0.5, log=True),\n        'num_leaves': trial.suggest_int('num_leaves', 8 , 256, log=True),\n        'subsample': trial.suggest_float(\"subsample\", 0.1, 1),\n        'colsample_bytree': trial.suggest_float(\"colsample_bytree\", 0.1, 1),\n        'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 100, log=True),\n        'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 100, log=True),\n        'min_child_samples':trial.suggest_int('min_child_samples', 20, 1000),\n        'min_split_gain': trial.suggest_float('min_split_gain',1e-8, 100,log=True), # gamma in xgboost,\n    }\n    \n    n_splits = 5\n    cv_class = StratifiedKFold if target_type == 'cat' else KFold\n    \n    split_loss = [None] * n_splits\n    for i, (train_index, test_index) in enumerate(cv_class(n_splits=n_splits, shuffle=True, random_state=1).split(X, y)):\n        split_loss[i] = fit_and_test_fold(target_type, X, y, train_index, test_index, params)\n    \n    # saving the individual fold holdout metrics \n    # uncomment this line if you don't want this\n    trial.set_user_attr('split_loss', split_loss)\n    \n    return np.mean(split_loss)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T22:16:00.604932Z","iopub.execute_input":"2024-05-09T22:16:00.605357Z","iopub.status.idle":"2024-05-09T22:16:00.617627Z","shell.execute_reply.started":"2024-05-09T22:16:00.605321Z","shell.execute_reply":"2024-05-09T22:16:00.616709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tune(X:pd.DataFrame, y:np.ndarray, target_type:str, task_id:str) -> Dict:\n    \n    obj = partial(optuna_objective, X=X, y=y, target_type=target_type)\n    study = optuna.create_study(\n        direction='minimize', \n        study_name=f'lightgbm_loss_{task_id}',\n        sampler=optuna.samplers.TPESampler(n_startup_trials=10, seed=1)\n    )\n        \n    study.optimize(obj, timeout=MAX_RUNTIME, n_trials=MAX_TRIALS)\n    \n    # save the runs\n    _ = joblib.dump(study, os.path.join(SAVE_DIR, f'lightgbm_{task_id}.pkl'))\n    \n    return study.best_params","metadata":{"execution":{"iopub.status.busy":"2024-05-09T22:16:03.269494Z","iopub.execute_input":"2024-05-09T22:16:03.270137Z","iopub.status.idle":"2024-05-09T22:16:03.279661Z","shell.execute_reply.started":"2024-05-09T22:16:03.270100Z","shell.execute_reply":"2024-05-09T22:16:03.277627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/bench-tab-v1/'\n\ntasks = json.load(open(f'{path}train/dict_task.json'))\nsuits = json.load(open(f'{path}train/dict_suit.json'))\n\nsuits = {'334': [361110, 361111], '335': [361093]} if DEBUG else suits\n\ndef prepare_sub(data, identifier):\n    data = pd.DataFrame(data)\n    data.columns = ['target']\n    data['ID'] =  identifier + '_' + data.index.astype(str)\n    columns = ['ID','target']\n    return data[columns]\n\nbest_params = {}\nsub = []\n\nfor t in suits:\n    print(t)\n    for suit in suits[t]:\n        \n        identifier = f'{t}_{suit}'\n        task_type = tasks[t]\n        var_type = task_type['features']\n        target_type = task_type['target']\n\n        X = (\n            pd.read_parquet(f'{path}train/{identifier}/data_train.parquet')\n            .rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))\n        )\n        X_test = (\n            pd.read_parquet(f'{path}train/{identifier}/data_test.parquet')\n            .rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))\n        )\n        y = pd.read_parquet(f'{path}train/{identifier}/target_train.parquet').values.ravel()\n        \n        names = json.load(open(f'{path}train/{identifier}/attribute_names.json'))\n        cat_indicator = json.load(open(f'{path}train/{identifier}/categorical_indicator.json'))\n        \n        # tune model\n        best_params[identifier] = tune(X, y, target_type, identifier)\n        \n        if target_type == 'num':\n            gbdt_model = LGBMRegressor(n_jobs=4, verbose=-1, **best_params[identifier])\n        else:\n            gbdt_model = LGBMClassifier(n_jobs=4, verbose=-1, **best_params[identifier])\n            \n            \n        _ = gbdt_model.fit(X, y)\n        \n        if target_type=='num':\n            preds = gbdt_model.predict(X_test)\n        else:\n            preds = gbdt_model.predict_proba(X_test)[:,1]\n            \n        preds = pd.DataFrame(preds, index = X_test.index)\n        \n        # appends empty tests / solutions\n        sub.append(prepare_sub(preds, identifier))\n        \n        print('')\n        print(f'{identifier=} {var_type=} {target_type=}')","metadata":{"execution":{"iopub.status.busy":"2024-05-09T22:16:08.960653Z","iopub.execute_input":"2024-05-09T22:16:08.961089Z","iopub.status.idle":"2024-05-09T22:19:31.489316Z","shell.execute_reply.started":"2024-05-09T22:16:08.961054Z","shell.execute_reply":"2024-05-09T22:19:31.488081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.concat(sub).reset_index(drop=True)\ndisplay(sub.head())\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T22:19:35.091316Z","iopub.execute_input":"2024-05-09T22:19:35.092140Z","iopub.status.idle":"2024-05-09T22:19:35.172086Z","shell.execute_reply.started":"2024-05-09T22:19:35.092100Z","shell.execute_reply":"2024-05-09T22:19:35.171085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save number of trees\nwith open(f'best_params_{MAX_RUNTIME // 60}min_{MAX_TRIALS}-trials.json', 'w') as f:\n    json.dump(best_params, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T22:20:02.449747Z","iopub.execute_input":"2024-05-09T22:20:02.450146Z","iopub.status.idle":"2024-05-09T22:20:02.456133Z","shell.execute_reply.started":"2024-05-09T22:20:02.450107Z","shell.execute_reply":"2024-05-09T22:20:02.455009Z"},"trusted":true},"execution_count":null,"outputs":[]}]}