{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67223,"databundleVersionId":7463838,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benchmarking Vowpal Wabbit for tabular data\n\nTabular data benchmark: ensemble of 5 un-tuned vowalpal wabbit.\nVowpal is a fast online learner.","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install ngboost","metadata":{"execution":{"iopub.status.busy":"2024-01-19T19:55:52.952548Z","iopub.execute_input":"2024-01-19T19:55:52.953941Z","iopub.status.idle":"2024-01-19T19:56:07.241117Z","shell.execute_reply.started":"2024-01-19T19:55:52.953894Z","shell.execute_reply":"2024-01-19T19:56:07.239586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from category_encoders import (\n    GLMMEncoder,\n    LeaveOneOutEncoder,\n    OneHotEncoder,\n    OrdinalEncoder,\n    TargetEncoder,\n    WOEEncoder,\n)\n\n\nfrom ngboost.distns import k_categorical, Exponential, LogNormal, Normal\nfrom ngboost import NGBClassifier, NGBRegressor\n\nimport numpy as np, pandas as pd\nimport os\nimport json\nfrom sklearn.metrics import log_loss, mean_squared_error\nfrom sklearn.ensemble import AdaBoostClassifier, IsolationForest, HistGradientBoostingClassifier, RandomForestClassifier, GradientBoostingRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-19T19:56:07.243793Z","iopub.execute_input":"2024-01-19T19:56:07.244173Z","iopub.status.idle":"2024-01-19T19:56:07.252521Z","shell.execute_reply.started":"2024-01-19T19:56:07.244137Z","shell.execute_reply":"2024-01-19T19:56:07.251460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_correlation(dataset, threshold=0.8):\n    col_corr = set()  # Set of all the names of deleted columns\n    corr_matrix = dataset.corr()\n    for i in range(len(corr_matrix.columns)):\n        for j in range(i):\n            if (corr_matrix.iloc[i, j] >= threshold) and (\n                corr_matrix.columns[j] not in col_corr\n            ):\n                colname = corr_matrix.columns[i]  # getting the name of column\n                col_corr.add(colname)\n                if colname in dataset.columns:\n                    del dataset[colname]  # deleting the column from the dataset\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2024-01-19T19:56:07.253910Z","iopub.execute_input":"2024-01-19T19:56:07.254272Z","iopub.status.idle":"2024-01-19T19:56:07.272681Z","shell.execute_reply.started":"2024-01-19T19:56:07.254241Z","shell.execute_reply":"2024-01-19T19:56:07.271506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/bench-tab-v1/'\n\ntasks = json.load(open(f'{path}train/dict_task.json'))\nsuits = json.load(open(f'{path}train/dict_suit.json'))\n\nDEBUG = False\nsuits = {'334': [361110, 361111], '335': [361093]} if DEBUG else suits\nn_seeds = 2 if DEBUG else 5\n\ndef prepare_sub(data, identifier):\n    data = pd.DataFrame(data)\n    data.columns = ['target']\n    data['ID'] =  identifier + '_' + data.index.astype(str)\n    columns = ['ID','target']\n    return data[columns]\n\nres_gbdt = {}\nsub = []\n\nfor t in suits:\n    print(t)\n    for suit in suits[t]:\n        \n        identifier = f'{t}_{suit}'\n        task_type = tasks[t]\n        var_type = task_type['features']\n        target_type = task_type['target']\n\n        X = pd.read_parquet(f'{path}train/{identifier}/data_train.parquet')\n        X_test = pd.read_parquet(f'{path}train/{identifier}/data_test.parquet')\n        \n        for column_name in X.columns:\n            # Check if the column has uint dtype\n            if X[column_name].dtype in ['uint', 'uint8', 'uint32', 'uint64']:\n                # Convert uint to int32\n                X[column_name] = X[column_name].astype('int32')\n                X_test[column_name] = X_test[column_name].astype('int32')\n        \n        y = pd.read_parquet(f'{path}train/{identifier}/target_train.parquet')\n        \n        names = json.load(open(f'{path}train/{identifier}/attribute_names.json'))\n        cat_indicator = json.load(open(f'{path}train/{identifier}/categorical_indicator.json'))\n        \n        #TO DO: random train / test split + custom metric\n        \n        preds = []\n        \n        for seed in range(n_seeds):\n            X_temp = X.copy()\n            X_test_temp = X_test.copy()\n            # target encode categories\n            enc = TargetEncoder(cols=X_temp.columns[cat_indicator].to_list())\n            X_temp.loc[:, X_temp.columns[cat_indicator].to_list()] = enc.fit_transform(\n                X_temp.loc[:, X_temp.columns[cat_indicator].to_list()], y\n            )\n            X_test_temp.loc[:, X_temp.columns[cat_indicator].to_list()] = enc.transform(X_test_temp.loc[:, X_temp.columns[cat_indicator].to_list()])\n            X_temp = remove_correlation(X_temp, threshold=0.8)\n            cols = X_temp.columns.to_list()\n            X_test_temp = X_test_temp.loc[:, cols]\n            \n            x_train, x_test, y_train, y_test = train_test_split(X_temp, y, test_size=0.20, random_state=seed+1)\n            \n            if target_type=='num':\n                y_train = y_train.astype(float)\n                y_test = y_test.astype(float)\n                \n                base_learner_choice = GradientBoostingRegressor(\n                                max_depth=7,\n                                n_estimators=300,\n                                n_iter_no_change=20,\n                                random_state=seed,\n                            )\n\n                model = NGBRegressor(\n                                Dist=LogNormal,\n                                n_estimators=300,\n                                #minibatch_frac=param[\"minibatch_frac\"],\n                                Base=base_learner_choice,\n                                learning_rate=0.1,\n                                natural_gradient=False,\n                            )\n\n                model.fit(\n                            x_train,\n                            y_train.values.ravel(),\n                            X_val=x_test,\n                            Y_val=y_test.values.ravel(),\n                            early_stopping_rounds=10,\n                            )\n            else:\n                y_train = y_train.astype(int)\n                y_test = y_test.astype(int)\n                \n                classes_weights_sample = class_weight.compute_sample_weight(\n                        class_weight=\"balanced\", y=y_train.values.ravel()\n                    )\n\n                base_learner_choice = GradientBoostingRegressor(\n                                max_depth=7,\n                                n_estimators=300,\n                                n_iter_no_change=20,\n                                random_state=seed,\n                            )\n\n                model = NGBClassifier(\n                                Dist=k_categorical(int(y_train.nunique())),\n                                n_estimators=300,\n                                #minibatch_frac=param[\"minibatch_frac\"],\n                                Base=base_learner_choice,\n                                learning_rate=0.1,\n                                natural_gradient=False,\n                            )\n\n                model.fit(\n                            x_train,\n                            y_train.values.ravel(),\n                            X_val=x_test,\n                            Y_val=y_test.values.ravel(),\n                            sample_weight=classes_weights_sample,\n                            early_stopping_rounds=10,\n                            )\n            \n            if target_type=='num':\n                preds.append(model.predict(X_test_temp))\n            else:\n                preds.append(model.predict_proba(X_test_temp)[:,1])\n            \n        preds = np.mean(np.array(preds),axis=0)\n\n        preds = pd.DataFrame(preds)\n        preds.index = X_test.index\n        \n        # appends empty tests / solutions\n        sub.append(prepare_sub(preds, identifier))\n        \n        print('')\n        print(f'{identifier=} {var_type=} {target_type=}')","metadata":{"execution":{"iopub.status.busy":"2024-01-19T20:04:11.174871Z","iopub.execute_input":"2024-01-19T20:04:11.175275Z","iopub.status.idle":"2024-01-19T20:04:32.343466Z","shell.execute_reply.started":"2024-01-19T20:04:11.175245Z","shell.execute_reply":"2024-01-19T20:04:32.341500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.concat(sub).reset_index(drop=True)\ndisplay(sub.head())\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-19T19:56:56.251340Z","iopub.status.idle":"2024-01-19T19:56:56.251776Z","shell.execute_reply.started":"2024-01-19T19:56:56.251574Z","shell.execute_reply":"2024-01-19T19:56:56.251594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}