{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":77585,"databundleVersionId":8490368,"sourceType":"competition"},{"sourceId":178708444,"sourceType":"kernelVersion"},{"sourceId":178885269,"sourceType":"kernelVersion"},{"sourceId":179810988,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":1701.301141,"end_time":"2024-05-11T09:55:57.636959","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-05-11T09:27:36.335818","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Only CatBoost for the Tabular Data Benchmark - 2nd Edition\n\nThis notebook validates and submits the predictions of 59 CatBoost models which were tuned individually. The out-of-fold scores are compared to HGB, XGBoost and LightGBM scores.\n\nReferences:\n- [TDB2 EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/tdb2-eda-which-makes-sense) (EDA notebook)\n- [TDB2 Only HGB](https://www.kaggle.com/code/ambrosm/tdb2-only-hgb), [TDB2 Only LightGBM](https://www.kaggle.com/code/ambrosm/tdb2-only-lightgbm), [TDB2 Only XGBoost](https://www.kaggle.com/code/ambrosm/tdb2-only-xgboost) (three notebooks to which we'll compare our scores)\n- [TDB2 Feature importance](https://www.kaggle.com/code/ambrosm/tdb2-feature-importance) (notebook which determines the important features)\n- [Tabular Data: are gbdt still outperforming DL?](https://www.kaggle.com/competitions/bench-tab-v1) (the predecessor to this competition)\n- [Tabular Data Benchmark - 2nd Edition](https://www.kaggle.com/competitions/bench-tab-v2) (this competition)\n- [Why do tree-based models still outperform deep learning on tabular data?](https://arxiv.org/pdf/2207.08815)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.006002,"end_time":"2024-05-11T09:27:39.414791","exception":false,"start_time":"2024-05-11T09:27:39.408789","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport matplotlib.pyplot as plt\nfrom collections import namedtuple, defaultdict, OrderedDict\nimport warnings\nimport catboost\nimport pickle\nfrom tqdm import tqdm\nimport datetime\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss, mean_squared_error\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.600495,"end_time":"2024-05-11T09:27:43.021178","exception":false,"start_time":"2024-05-11T09:27:39.420683","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T17:01:14.976056Z","iopub.execute_input":"2024-05-29T17:01:14.976855Z","iopub.status.idle":"2024-05-29T17:01:16.654986Z","shell.execute_reply.started":"2024-05-29T17:01:14.976812Z","shell.execute_reply":"2024-05-29T17:01:16.653929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configuration\n# number of repeats for ensembling\nn_repeats = 5\n","metadata":{"papermill":{"duration":0.01463,"end_time":"2024-05-11T09:27:43.041281","exception":false,"start_time":"2024-05-11T09:27:43.026651","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T17:01:16.657004Z","iopub.execute_input":"2024-05-29T17:01:16.657531Z","iopub.status.idle":"2024-05-29T17:01:16.662250Z","shell.execute_reply.started":"2024-05-29T17:01:16.657500Z","shell.execute_reply":"2024-05-29T17:01:16.661130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the data\n\nWe read all the data files into the dictionary `datasets`.","metadata":{"papermill":{"duration":0.005132,"end_time":"2024-05-11T09:27:43.051820","exception":false,"start_time":"2024-05-11T09:27:43.046688","status":"completed"},"tags":[]}},{"cell_type":"code","source":"path = '/kaggle/input/bench-tab-v2/'\ntask_types = json.load(open(f'{path}train/dict_task.json'))\nsuits = json.load(open(f'{path}train/dict_suit.json'))\n\nDataset = namedtuple('Dataset', field_names=['X', 'X_te', 'y', 'names', 'cat_indicator', 'var_type', 'target_type'])\n\nsize_list = []\ndatasets = {}\nfor task_type_num in suits: # 4 entries\n    for suit in suits[task_type_num]: # 59 entries\n        \n        identifier = f'{task_type_num}_{suit}'\n        task_type = task_types[task_type_num]\n        var_type = task_type['features'] # 'num' or 'num+cat'\n        target_type = task_type['target'] # 'num' or 'cat'\n\n        # Read the three data files\n        X = pd.read_parquet(f'{path}train/{identifier}/data_train.parquet')\n        X_te = pd.read_parquet(f'{path}train/{identifier}/data_test.parquet')\n        y = pd.read_parquet(f'{path}train/{identifier}/target_train.parquet')\n        assert len(X) == len(y)\n        \n        # Check for duplicate rows\n        n_unique_tr = len(X.drop_duplicates())\n        n_unique_te = len(X_te.drop_duplicates())\n        \n        # Read the two metadata files\n        names = json.load(open(f'{path}train/{identifier}/attribute_names.json'))\n        X.columns = [s.replace(':', '') for s in X.columns] # LightGBM doesn't like colons\n        X_te.columns = [s.replace(':', '') for s in X_te.columns]\n        names = [s.replace(':', '') for s in names]\n        assert (names == X.columns).all()\n        assert (names == X_te.columns).all()\n        cat_indicator = json.load(open(f'{path}train/{identifier}/categorical_indicator.json'))\n        \n        size_list.append((identifier, var_type, target_type, len(X), len(X_te), len(names), n_unique_tr, n_unique_te))\n        datasets[identifier] = Dataset(X, X_te, y, names, cat_indicator, var_type, target_type)\nsizes = pd.DataFrame(size_list, columns=['Identifier', 'var_type', 'target_type', 'n_samples_tr', 'n_samples_te', 'n_features', 'n_unique_tr', 'n_unique_te'])\ndel X, X_te, y, names, identifier, cat_indicator\n","metadata":{"papermill":{"duration":17.760678,"end_time":"2024-05-11T09:28:00.817845","exception":false,"start_time":"2024-05-11T09:27:43.057167","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T17:01:16.664107Z","iopub.execute_input":"2024-05-29T17:01:16.664469Z","iopub.status.idle":"2024-05-29T17:01:36.569507Z","shell.execute_reply.started":"2024-05-29T17:01:16.664441Z","shell.execute_reply":"2024-05-29T17:01:36.568337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameters\n\nHyperparameters were determined in 59 Optuna studies with 30 trials each. This optimization took seven hours. The hyperparameters are hidden in the following cell.","metadata":{"papermill":{"duration":0.004999,"end_time":"2024-05-11T09:28:00.828312","exception":false,"start_time":"2024-05-11T09:28:00.823313","status":"completed"},"tags":[]}},{"cell_type":"code","source":"cb_params = {'334_361110': ({'grow_policy': 'Lossguide', 'n_estimators': 1303, 'learning_rate': 0.018152952915925353, 'l2_leaf_reg': 2.417729854571641, 'max_leaves': 225, 'max_depth': 9, 'min_child_samples': 8, 'colsample_bylevel': 0.9074451591847135, 'random_strength': 0.038664514661312266, 'boost_from_average': True, 'verbose': False}, 0.2558056214752239), '334_361111': ({'grow_policy': 'Lossguide', 'n_estimators': 1156, 'learning_rate': 0.005183007979649811, 'l2_leaf_reg': 0.14903068839335853, 'max_depth': 8, 'max_leaves': 188, 'min_child_samples': 27, 'colsample_bylevel': 0.5949415192813666, 'random_strength': 0.21760922262485974, 'boost_from_average': True, 'verbose': False}, 0.6350455878013712), '334_361113': ({'grow_policy': 'Lossguide', 'n_estimators': 1109, 'learning_rate': 0.17281333707691207, 'l2_leaf_reg': 6.037612098112019, 'max_depth': 10, 'max_leaves': 313, 'min_child_samples': 35, 'colsample_bylevel': 0.8544221455501531, 'random_strength': 0.14804715727433035, 'boost_from_average': True, 'verbose': False}, 0.08132958740887718), '334_361282': ({'grow_policy': 'Lossguide', 'n_estimators': 1231, 'learning_rate': 0.004475128751856959, 'l2_leaf_reg': 4.610461458719303, 'max_depth': 7, 'max_leaves': 90, 'min_child_samples': 96, 'colsample_bylevel': 0.6002347930143196, 'random_strength': 0.3830567173550909, 'boost_from_average': True, 'verbose': False}, 0.6160669448125828), '334_361283': ({'grow_policy': 'Lossguide', 'n_estimators': 352, 'learning_rate': 0.01264652247041063, 'l2_leaf_reg': 0.6470105585767909, 'max_depth': 6, 'max_leaves': 19, 'min_child_samples': 17, 'colsample_bylevel': 0.8290473519281546, 'random_strength': 0.257362915492169, 'boost_from_average': True, 'verbose': False}, 0.5501060561436003), '334_361285': ({'grow_policy': 'Lossguide', 'n_estimators': 842, 'learning_rate': 0.05053352946755655, 'l2_leaf_reg': 4.913833224624566, 'max_depth': 9, 'max_leaves': 298, 'min_child_samples': 4, 'colsample_bylevel': 0.8085797331924243, 'random_strength': 0.32670814236272505, 'boost_from_average': True, 'verbose': False}, 0.4054329905314882), '334_361286': ({'grow_policy': 'Lossguide', 'n_estimators': 260, 'learning_rate': 0.017565862488252097, 'l2_leaf_reg': 9.954301066093162, 'max_depth': 6, 'max_leaves': 138, 'min_child_samples': 100, 'colsample_bylevel': 0.7024829920903417, 'random_strength': 0.9486982595223608, 'boost_from_average': True, 'verbose': False}, 0.6049610425650458), '335_361093': ({'grow_policy': 'Lossguide', 'n_estimators': 206, 'learning_rate': 0.22512907828208276, 'l2_leaf_reg': 9.16814584188264, 'max_depth': 9, 'max_leaves': 377, 'min_child_samples': 83, 'colsample_bylevel': 0.8671402936229243, 'random_strength': 0.9512207253106402, 'boost_from_average': True, 'verbose': False}, 0.0047615413102267265), '335_361094': ({'grow_policy': 'Lossguide', 'n_estimators': 400, 'learning_rate': 0.06650129332498497, 'l2_leaf_reg': 1.4783760211481707, 'max_leaves': 24, 'min_child_samples': 12, 'colsample_bylevel': 0.981409369317944, 'random_strength': 0.04962707061746156, 'boost_from_average': True, 'verbose': False}, 0.00672010771523162), '335_361096': ({'grow_policy': 'Lossguide', 'n_estimators': 400, 'learning_rate': 0.1055828935605486, 'l2_leaf_reg': 4.898435439994307, 'max_depth': 9, 'max_leaves': 155, 'min_child_samples': 8, 'colsample_bylevel': 0.9133025718009481, 'random_strength': 0.22375354201214176, 'boost_from_average': True, 'verbose': False}, 0.007541554076498591), '335_361097': ({'grow_policy': 'Lossguide', 'n_estimators': 764, 'learning_rate': 0.009123535087510215, 'l2_leaf_reg': 8.802176668449487, 'max_depth': 6, 'max_leaves': 5, 'min_child_samples': 99, 'colsample_bylevel': 0.9656331675318655, 'random_strength': 0.37480776939302685, 'boost_from_average': True, 'verbose': False}, 59.888809732801676), '335_361098': ({'grow_policy': 'Lossguide', 'n_estimators': 298, 'learning_rate': 0.039856615923332214, 'l2_leaf_reg': 4.098025698509235, 'max_leaves': 158, 'min_child_samples': 2, 'colsample_bylevel': 0.8518879169253278, 'random_strength': 0.15697406279402276, 'boost_from_average': True, 'verbose': False}, 0.0058803422202505055), '335_361099': ({'grow_policy': 'Lossguide', 'n_estimators': 410, 'learning_rate': 0.04404950290383797, 'l2_leaf_reg': 0.9945241265258257, 'max_depth': 9, 'max_leaves': 151, 'min_child_samples': 7, 'colsample_bylevel': 0.7563411856704751, 'random_strength': 0.8033150661639273, 'boost_from_average': True, 'verbose': False}, 1821.136672975516), '335_361101': ({'grow_policy': 'Lossguide', 'n_estimators': 875, 'learning_rate': 0.07136543536921096, 'l2_leaf_reg': 0.5787810637891406, 'max_depth': 9, 'max_leaves': 212, 'min_child_samples': 7, 'colsample_bylevel': 0.8247021672756437, 'random_strength': 0.16305091659435655, 'boost_from_average': True, 'verbose': False}, 0.11870888815557532), '335_361102': ({'grow_policy': 'Lossguide', 'n_estimators': 1303, 'learning_rate': 0.023092305714617786, 'l2_leaf_reg': 0.2305493668529377, 'max_depth': 6, 'max_leaves': 61, 'min_child_samples': 13, 'colsample_bylevel': 0.7445900981534737, 'random_strength': 0.714804561714395, 'boost_from_average': True, 'verbose': False}, 0.02872553955075606), '335_361103': ({'grow_policy': 'Lossguide', 'n_estimators': 1166, 'learning_rate': 0.051225474730360654, 'l2_leaf_reg': 2.3522510659608797, 'max_depth': 8, 'max_leaves': 219, 'min_child_samples': 36, 'colsample_bylevel': 0.808551419420558, 'random_strength': 0.9147851364641972, 'boost_from_average': True, 'verbose': False}, 0.13448347988636078), '335_361104': ({'grow_policy': 'Lossguide', 'n_estimators': 387, 'learning_rate': 0.2751596801954292, 'l2_leaf_reg': 6.866924452254358, 'max_leaves': 181, 'min_child_samples': 30, 'colsample_bylevel': 0.8608788314081757, 'random_strength': 0.18384942733429738, 'boost_from_average': True, 'verbose': False}, 0.0002843202238033592), '335_361287': ({'grow_policy': 'Lossguide', 'n_estimators': 549, 'learning_rate': 0.005558833886550722, 'l2_leaf_reg': 2.2068755362516868, 'max_depth': 8, 'max_leaves': 123, 'min_child_samples': 39, 'colsample_bylevel': 0.655549848521531, 'random_strength': 0.11954931903204236, 'boost_from_average': True, 'verbose': False}, 0.0007059120303024007), '335_361288': ({'grow_policy': 'Lossguide', 'n_estimators': 358, 'learning_rate': 0.038173079988225084, 'l2_leaf_reg': 6.185935849305832, 'max_depth': 8, 'max_leaves': 38, 'min_child_samples': 58, 'colsample_bylevel': 0.8998474666867115, 'random_strength': 0.749432411374614, 'boost_from_average': True, 'verbose': False}, 4.427317223435601), '335_361289': ({'grow_policy': 'Lossguide', 'n_estimators': 106, 'learning_rate': 0.16655184831109732, 'l2_leaf_reg': 0.13434233281828012, 'max_depth': 8, 'max_leaves': 125, 'min_child_samples': 6, 'colsample_bylevel': 0.8516273014956208, 'random_strength': 0.9822955934400149, 'boost_from_average': True, 'verbose': False}, 145001.80579073008), '335_361291': ({'grow_policy': 'Lossguide', 'n_estimators': 400, 'learning_rate': 0.43773595202466914, 'l2_leaf_reg': 0.9126862737932364, 'max_leaves': 115, 'min_child_samples': 1, 'colsample_bylevel': 0.8619291800770132, 'random_strength': 0.8641357768017038, 'boost_from_average': True, 'verbose': False}, 8.7379702756405), '335_361292': ({'grow_policy': 'Lossguide', 'n_estimators': 1405, 'learning_rate': 0.03701596054698621, 'l2_leaf_reg': 8.096492586462459, 'max_depth': 9, 'max_leaves': 285, 'min_child_samples': 14, 'colsample_bylevel': 0.8105575773613305, 'random_strength': 0.7394860776033326, 'boost_from_average': True, 'verbose': False}, 0.2870510714501575), '335_361293': ({'grow_policy': 'Lossguide', 'n_estimators': 350, 'learning_rate': 0.1023744838609053, 'l2_leaf_reg': 9.48746169008554, 'max_leaves': 52, 'min_child_samples': 96, 'colsample_bylevel': 0.8768654728277465, 'random_strength': 0.3530138566471983, 'boost_from_average': True, 'verbose': False}, 3.635228354327448), '335_361294': ({'grow_policy': 'Lossguide', 'n_estimators': 680, 'learning_rate': 0.012113840715608767, 'l2_leaf_reg': 6.9146791225539825, 'max_depth': 6, 'max_leaves': 58, 'min_child_samples': 80, 'colsample_bylevel': 0.985423987532758, 'random_strength': 0.9748064794455518, 'boost_from_average': True, 'verbose': False}, 0.006786224737550314), '336_361072': ({'grow_policy': 'Lossguide', 'n_estimators': 963, 'learning_rate': 0.02037349983737404, 'l2_leaf_reg': 0.9340333336868332, 'max_depth': 7, 'max_leaves': 85, 'min_child_samples': 78, 'colsample_bylevel': 0.8737011173340538, 'random_strength': 0.5724286417874533, 'boost_from_average': True, 'verbose': False}, 5.0176285656896855), '336_361073': ({'grow_policy': 'Lossguide', 'n_estimators': 1331, 'learning_rate': 0.02277386838995499, 'l2_leaf_reg': 1.314059561539665, 'max_depth': 9, 'max_leaves': 168, 'min_child_samples': 2, 'colsample_bylevel': 0.6971771293510334, 'random_strength': 0.32151807113223074, 'boost_from_average': True, 'verbose': False}, 17.299799889855443), '336_361074': ({'grow_policy': 'Lossguide', 'n_estimators': 1328, 'learning_rate': 0.065375314702084, 'l2_leaf_reg': 5.896067678043449, 'max_depth': 6, 'max_leaves': 27, 'min_child_samples': 54, 'colsample_bylevel': 0.6730421021070181, 'random_strength': 0.7996602696871706, 'boost_from_average': True, 'verbose': False}, 4.763862812659464e-06), '336_361076': ({'grow_policy': 'Lossguide', 'n_estimators': 388, 'learning_rate': 0.10039361396830425, 'l2_leaf_reg': 1.9391875751737995, 'max_depth': 9, 'max_leaves': 191, 'min_child_samples': 2, 'colsample_bylevel': 0.8871908297580261, 'random_strength': 0.6590472546140091, 'boost_from_average': True, 'verbose': False}, 0.37896215658298477), '336_361077': ({'grow_policy': 'Lossguide', 'n_estimators': 426, 'learning_rate': 0.05927671391260587, 'l2_leaf_reg': 0.8160848740230167, 'max_depth': 9, 'max_leaves': 398, 'min_child_samples': 45, 'colsample_bylevel': 0.7741357121674053, 'random_strength': 0.03480902548013945, 'boost_from_average': True, 'verbose': False}, 2.526356311786757e-08), '336_361078': ({'grow_policy': 'Lossguide', 'n_estimators': 1457, 'learning_rate': 0.030647748481469925, 'l2_leaf_reg': 2.51499267989298, 'max_depth': 7, 'max_leaves': 122, 'min_child_samples': 24, 'colsample_bylevel': 0.8837260612574427, 'random_strength': 0.5392458442522147, 'boost_from_average': True, 'verbose': False}, 0.04742950646673808), '336_361079': ({'grow_policy': 'Lossguide', 'n_estimators': 304, 'learning_rate': 0.12520762536383012, 'l2_leaf_reg': 5.944612188236299, 'max_depth': 8, 'max_leaves': 132, 'min_child_samples': 4, 'colsample_bylevel': 0.7564106688816782, 'random_strength': 0.3735342914604284, 'boost_from_average': True, 'verbose': False}, 0.37534047219434713), '336_361080': ({'grow_policy': 'Lossguide', 'n_estimators': 1016, 'learning_rate': 0.01464792112736329, 'l2_leaf_reg': 2.14699339095138, 'max_depth': 8, 'max_leaves': 208, 'min_child_samples': 44, 'colsample_bylevel': 0.8583206877483865, 'random_strength': 0.1402993234361768, 'boost_from_average': True, 'verbose': False}, 0.05467848041078435), '336_361081': ({'grow_policy': 'Lossguide', 'n_estimators': 619, 'learning_rate': 0.0740001926318086, 'l2_leaf_reg': 5.844568367373893, 'max_depth': 6, 'max_leaves': 44, 'min_child_samples': 76, 'colsample_bylevel': 0.5267365159653126, 'random_strength': 0.5404761807537566, 'boost_from_average': True, 'verbose': False}, 0.0017497350339577534), '336_361082': ({'grow_policy': 'Lossguide', 'n_estimators': 212, 'learning_rate': 0.08324909373879791, 'l2_leaf_reg': 1.3795281173565739, 'max_depth': 7, 'max_leaves': 47, 'min_child_samples': 11, 'colsample_bylevel': 0.6524113235183018, 'random_strength': 0.2646199795358823, 'boost_from_average': True, 'verbose': False}, 9888.752658406007), '336_361083': ({'grow_policy': 'Lossguide', 'n_estimators': 757, 'learning_rate': 0.17387797844166702, 'l2_leaf_reg': 9.214236783140223, 'max_depth': 6, 'max_leaves': 62, 'min_child_samples': 68, 'colsample_bylevel': 0.8492525277487806, 'random_strength': 0.4185831974030806, 'boost_from_average': True, 'verbose': False}, 0.1299434170319554), '336_361084': ({'grow_policy': 'Lossguide', 'n_estimators': 1432, 'learning_rate': 0.024260602102274965, 'l2_leaf_reg': 3.143930342569763, 'max_depth': 9, 'max_leaves': 31, 'min_child_samples': 37, 'colsample_bylevel': 0.5382270271623664, 'random_strength': 0.7192705698120752, 'boost_from_average': True, 'verbose': False}, 0.03120111823976248), '336_361085': ({'grow_policy': 'Lossguide', 'n_estimators': 344, 'learning_rate': 0.05473096730963149, 'l2_leaf_reg': 0.9014972603314464, 'max_depth': 9, 'max_leaves': 300, 'min_child_samples': 12, 'colsample_bylevel': 0.5906806662876647, 'random_strength': 0.4047298326525569, 'boost_from_average': True, 'verbose': False}, 0.0005001921279884292), '336_361086': ({'grow_policy': 'Lossguide', 'n_estimators': 400, 'learning_rate': 0.016629679947240496, 'l2_leaf_reg': 9.796022718529295, 'max_leaves': 95, 'min_child_samples': 88, 'colsample_bylevel': 0.9898234382818658, 'random_strength': 0.5837565405977065, 'boost_from_average': True, 'verbose': False}, 0.006813933509009619), '336_361087': ({'grow_policy': 'Lossguide', 'n_estimators': 568, 'learning_rate': 0.03137090860012121, 'l2_leaf_reg': 3.1959711462079685, 'max_depth': 8, 'max_leaves': 120, 'min_child_samples': 15, 'colsample_bylevel': 0.5524028747727274, 'random_strength': 0.5210490337093386, 'boost_from_average': True, 'verbose': False}, 0.021885264445194216), '336_361088': ({'grow_policy': 'Lossguide', 'n_estimators': 360, 'learning_rate': 0.08846653539828486, 'l2_leaf_reg': 0.4096087117096734, 'max_depth': 8, 'max_leaves': 190, 'min_child_samples': 13, 'colsample_bylevel': 0.6223343472206981, 'random_strength': 0.6084026824481912, 'boost_from_average': True, 'verbose': False}, 91.37197161350565), '336_361279': ({'grow_policy': 'Lossguide', 'n_estimators': 537, 'learning_rate': 0.01941073480745395, 'l2_leaf_reg': 4.593984291009677, 'max_depth': 8, 'max_leaves': 168, 'min_child_samples': 23, 'colsample_bylevel': 0.5039569929692119, 'random_strength': 0.6964556957035023, 'boost_from_average': True, 'verbose': False}, 0.000684834244590671), '336_361280': ({'grow_policy': 'Lossguide', 'n_estimators': 307, 'learning_rate': 0.027539974927151423, 'l2_leaf_reg': 4.914801024578725, 'max_leaves': 131, 'min_child_samples': 7, 'colsample_bylevel': 0.7626413295081972, 'random_strength': 0.8956385507482499, 'boost_from_average': True, 'verbose': False}, 4.409883222998832), '336_361281': ({'grow_policy': 'Lossguide', 'n_estimators': 392, 'learning_rate': 0.17338824237417533, 'l2_leaf_reg': 2.456615954559103, 'max_leaves': 167, 'min_child_samples': 87, 'colsample_bylevel': 0.5825430724734337, 'random_strength': 0.4173378763963681, 'boost_from_average': True, 'verbose': False}, 9.268996373511145), '337_361055': ({'grow_policy': 'Lossguide', 'n_estimators': 973, 'learning_rate': 0.005550829492616134, 'l2_leaf_reg': 4.765879120281289, 'max_depth': 6, 'max_leaves': 23, 'min_child_samples': 15, 'colsample_bylevel': 0.9620790673420395, 'random_strength': 0.8423473036479178, 'boost_from_average': True, 'verbose': False}, 0.47490732430402505), '337_361060': ({'grow_policy': 'Lossguide', 'n_estimators': 323, 'learning_rate': 0.17473973854446298, 'l2_leaf_reg': 2.460296593555499, 'max_leaves': 198, 'min_child_samples': 30, 'colsample_bylevel': 0.8031047239790801, 'random_strength': 0.02599612478575807, 'boost_from_average': True, 'verbose': False}, 0.2827257916786396), '337_361061': ({'grow_policy': 'Lossguide', 'n_estimators': 1208, 'learning_rate': 0.15137153636931008, 'l2_leaf_reg': 1.8634502352413937, 'max_depth': 10, 'max_leaves': 267, 'min_child_samples': 50, 'colsample_bylevel': 0.761002295958228, 'random_strength': 0.05678090000281619, 'boost_from_average': True, 'verbose': False}, 0.10806744935372185), '337_361062': ({'grow_policy': 'Lossguide', 'n_estimators': 1025, 'learning_rate': 0.009785831256839315, 'l2_leaf_reg': 5.129110604638575, 'max_depth': 8, 'max_leaves': 154, 'min_child_samples': 45, 'colsample_bylevel': 0.8658577876779148, 'random_strength': 0.9574181868044566, 'boost_from_average': True, 'verbose': False}, 0.041098615321830206), '337_361063': ({'grow_policy': 'Lossguide', 'n_estimators': 741, 'learning_rate': 0.013302655941434116, 'l2_leaf_reg': 2.1416012726817857, 'max_depth': 6, 'max_leaves': 53, 'min_child_samples': 35, 'colsample_bylevel': 0.8783329019530964, 'random_strength': 0.9265569230739127, 'boost_from_average': True, 'verbose': False}, 0.29322988882051604), '337_361065': ({'grow_policy': 'Lossguide', 'n_estimators': 664, 'learning_rate': 0.008266240239520418, 'l2_leaf_reg': 2.648514367944956, 'max_depth': 9, 'max_leaves': 375, 'min_child_samples': 42, 'colsample_bylevel': 0.8292267492592665, 'random_strength': 0.443262370974714, 'boost_from_average': True, 'verbose': False}, 0.3281360164260004), '337_361066': ({'grow_policy': 'Lossguide', 'n_estimators': 1371, 'learning_rate': 0.005342081721620898, 'l2_leaf_reg': 3.0068387180131664, 'max_depth': 9, 'max_leaves': 300, 'min_child_samples': 81, 'colsample_bylevel': 0.6117136385006247, 'random_strength': 0.6359925069566883, 'boost_from_average': True, 'verbose': False}, 0.42870927186996294), '337_361068': ({'grow_policy': 'Lossguide', 'n_estimators': 1407, 'learning_rate': 0.0070953902547551, 'l2_leaf_reg': 2.182677348883545, 'max_depth': 8, 'max_leaves': 186, 'min_child_samples': 81, 'colsample_bylevel': 0.6477813218053399, 'random_strength': 0.1449333029189267, 'boost_from_average': True, 'verbose': False}, 0.15543644354450495), '337_361069': ({'grow_policy': 'Lossguide', 'n_estimators': 200, 'learning_rate': 0.24196652260435864, 'l2_leaf_reg': 5.620880778450278, 'max_leaves': 128, 'min_child_samples': 93, 'colsample_bylevel': 0.9341036940096528, 'random_strength': 0.3503917623164731, 'boost_from_average': True, 'verbose': False}, 0.5178943321145412), '337_361070': ({'grow_policy': 'Lossguide', 'n_estimators': 920, 'learning_rate': 0.006831048368597904, 'l2_leaf_reg': 9.00848795173599, 'max_depth': 9, 'max_leaves': 265, 'min_child_samples': 37, 'colsample_bylevel': 0.895549176638265, 'random_strength': 0.15831222354538832, 'boost_from_average': True, 'verbose': False}, 0.6320747148795354), '337_361273': ({'grow_policy': 'Lossguide', 'n_estimators': 200, 'learning_rate': 0.03670333737035239, 'l2_leaf_reg': 6.313929243275616, 'max_depth': 6, 'max_leaves': 23, 'min_child_samples': 30, 'colsample_bylevel': 0.9650628672677911, 'random_strength': 0.7867569863342379, 'boost_from_average': True, 'verbose': False}, 0.6571084356110268), '337_361274': ({'grow_policy': 'Lossguide', 'n_estimators': 914, 'learning_rate': 0.014197960612922108, 'l2_leaf_reg': 4.354286141488396, 'max_depth': 9, 'max_leaves': 336, 'min_child_samples': 13, 'colsample_bylevel': 0.9581283304010236, 'random_strength': 0.5071156866064424, 'boost_from_average': True, 'verbose': False}, 0.43680287895202513), '337_361275': ({'grow_policy': 'Lossguide', 'n_estimators': 200, 'learning_rate': 0.034572258753893886, 'l2_leaf_reg': 8.972388171696506, 'max_depth': 7, 'max_leaves': 17, 'min_child_samples': 83, 'colsample_bylevel': 0.8415597510583984, 'random_strength': 0.9355673095343526, 'boost_from_average': True, 'verbose': False}, 0.5582426378105569), '337_361276': ({'grow_policy': 'Lossguide', 'n_estimators': 1330, 'learning_rate': 0.0055482323687613775, 'l2_leaf_reg': 1.1469236449488913, 'max_depth': 7, 'max_leaves': 94, 'min_child_samples': 20, 'colsample_bylevel': 0.5245040178051372, 'random_strength': 0.8604736023759528, 'boost_from_average': True, 'verbose': False}, 0.5120509336449637), '337_361277': ({'grow_policy': 'Lossguide', 'n_estimators': 614, 'learning_rate': 0.036473961010086146, 'l2_leaf_reg': 9.301629631864014, 'max_depth': 7, 'max_leaves': 27, 'min_child_samples': 1, 'colsample_bylevel': 0.8085626405929536, 'random_strength': 0.7290701264773677, 'boost_from_average': True, 'verbose': False}, 0.23044974025615536), '337_361278': ({'grow_policy': 'Lossguide', 'n_estimators': 200, 'learning_rate': 0.02332044450762997, 'l2_leaf_reg': 6.648288846038197, 'max_depth': 9, 'max_leaves': 23, 'min_child_samples': 91, 'colsample_bylevel': 0.9776361082264067, 'random_strength': 0.9169692799860804, 'boost_from_average': True, 'verbose': False}, 0.5553600951405597)}\nimportant_features = {'334_361110': ['date', 'day', 'period', 'nswprice', 'nswdemand', 'vicprice', 'vicdemand', 'transfer'], '334_361111': ['lineNo', 'assgNo', 'P1stFixation', 'P2stFixation', 'prevFixDur', 'firstfixDur', 'firstPassFixDur', 'nextFixDur', 'firstSaccLen', 'lastSaccLen', 'prevFixPos', 'landingPos', 'totalFixDur', 'regressLen', 'nextWordRegress', 'regressDur', 'pupilDiamMax', 'pupilDiamLag', 'timePrtctg', 'titleNo', 'wordNo'], '334_361113': ['Elevation', 'Aspect', 'Slope', 'Horizontal_Distance_To_Hydrology', 'Vertical_Distance_To_Hydrology', 'Horizontal_Distance_To_Roadways', 'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm', 'Horizontal_Distance_To_Fire_Points', 'Wilderness_Area1', 'Wilderness_Area2', 'Wilderness_Area3', 'Wilderness_Area4', 'Soil_Type2', 'Soil_Type3', 'Soil_Type4', 'Soil_Type7', 'Soil_Type8', 'Soil_Type9', 'Soil_Type10', 'Soil_Type11', 'Soil_Type12', 'Soil_Type13', 'Soil_Type16', 'Soil_Type17', 'Soil_Type18', 'Soil_Type19', 'Soil_Type20', 'Soil_Type21', 'Soil_Type22', 'Soil_Type23', 'Soil_Type24', 'Soil_Type25', 'Soil_Type26', 'Soil_Type27', 'Soil_Type28', 'Soil_Type29', 'Soil_Type30', 'Soil_Type31', 'Soil_Type32', 'Soil_Type33', 'Soil_Type34', 'Soil_Type35', 'Soil_Type36', 'Soil_Type38', 'Soil_Type39', 'Soil_Type40'], '334_361282': ['V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V11', 'V13', 'V19', 'V30', 'V33', 'V35', 'V36', 'V41', 'V43', 'V47', 'V59', 'V63', 'V72', 'V75'], '334_361283': ['x1', 'x2', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10', 'x11', 'x12', 'x13', 'x14', 'x15', 'x16', 'x18', 'x19', 'x20', 'x21', 'x22', 'x23'], '334_361285': ['Vehicle_Reference_df_res', 'Vehicle_Type', 'Vehicle_Manoeuvre', 'Hit_Object_in_Carriageway', 'Hit_Object_off_Carriageway', 'Was_Vehicle_Left_Hand_Drive?', 'Age_of_Driver', 'Age_Band_of_Driver', 'Engine_Capacity_(CC)', 'Propulsion_Code', 'Age_of_Vehicle', 'Location_Easting_OSGR', 'Location_Northing_OSGR', 'Longitude', 'Latitude', 'Police_Force', 'Number_of_Vehicles', 'Number_of_Casualties', 'Local_Authority_(District)', '1st_Road_Number', '2nd_Road_Number', 'Urban_or_Rural_Area', 'Vehicle_Reference_df', 'Casualty_Reference', 'Sex_of_Casualty', 'Age_of_Casualty', 'Age_Band_of_Casualty', 'Pedestrian_Location', 'Pedestrian_Movement', 'Casualty_Type', 'Casualty_IMD_Decile'], '334_361286': ['sex', 'age', 'priors_count', 'age_cat_25-45', 'age_cat_Greaterthan45', 'age_cat_Lessthan25', 'race_African-American', 'race_Caucasian', 'c_charge_degree_F', 'c_charge_degree_M'], '335_361093': ['Actions_taken', 'Liberal', 'Unconstitutional', 'Precedent_alteration', 'Unanimous', 'Year_of_decision', 'Lower_court_disagreement'], '335_361094': ['northing', 'easting', 'resistivity', 'isns'], '335_361096': ['carat', 'cut', 'color', 'clarity', 'depth', 'table', 'x', 'y', 'z'], '335_361097': ['X3', 'X14', 'X19', 'X29', 'X31', 'X35', 'X37', 'X46', 'X47', 'X48', 'X49', 'X50', 'X52', 'X54', 'X61', 'X65', 'X75', 'X76', 'X80', 'X82', 'X96', 'X98', 'X100', 'X104', 'X111', 'X113', 'X115', 'X117', 'X118', 'X119', 'X120', 'X127', 'X131', 'X132', 'X134', 'X136', 'X139', 'X141', 'X147', 'X148', 'X150', 'X152', 'X156', 'X158', 'X164', 'X168', 'X176', 'X179', 'X181', 'X182', 'X187', 'X189', 'X191', 'X201', 'X208', 'X213', 'X215', 'X218', 'X221', 'X225', 'X226', 'X232', 'X234', 'X238', 'X240', 'X261', 'X263', 'X264', 'X265', 'X267', 'X273', 'X275', 'X283', 'X294', 'X311', 'X313', 'X314', 'X315', 'X316', 'X321', 'X322', 'X324', 'X327', 'X334', 'X340', 'X345', 'X349', 'X350', 'X354', 'X368', 'X376', 'X377'], '335_361098': ['city', 'area', 'rooms', 'bathroom', 'parking_spaces', 'animal', 'furniture', 'hoa_(BRL)', 'rent_amount_(BRL)', 'property_tax_(BRL)', 'fire_insurance_(BRL)'], '335_361099': ['season', 'year', 'month', 'hour', 'holiday', 'workingday', 'weather', 'temp', 'feel_temp', 'humidity', 'windspeed'], '335_361101': ['VendorID', 'store_and_fwd_flag', 'RatecodeID', 'passenger_count', 'extra', 'mta_tax', 'tolls_amount', 'improvement_surcharge', 'total_amount', 'trip_type', 'lpep_pickup_datetime_day', 'lpep_pickup_datetime_hour', 'lpep_pickup_datetime_minute', 'lpep_dropoff_datetime_day', 'lpep_dropoff_datetime_hour', 'lpep_dropoff_datetime_minute'], '335_361102': ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'waterfront', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15', 'date_year', 'date_month', 'date_day'], '335_361103': ['Hour', 'Month', 'DayofWeek', 'Environment.Type', 'Altitude..m.', 'PM.sub.2.5..sub..particulate.matter..Hourly.measured.'], '335_361104': ['KWG', 'KWI', 'STRM', 'STRN', 'SA', 'SB', 'Run2', 'Run3', 'Run4'], '335_361287': ['oz2', 'oz3', 'oz4', 'oz5', 'oz8', 'oz10', 'oz11', 'oz13', 'oz14', 'oz15', 'oz16', 'oz17', 'oz18', 'oz19', 'oz21', 'oz30', 'oz31', 'oz36', 'oz39', 'oz40', 'oz41', 'oz42', 'oz43', 'oz44', 'oz45', 'oz46', 'oz47', 'oz48', 'oz50', 'oz52', 'oz53', 'oz56', 'oz59', 'oz61', 'oz62', 'oz64', 'oz65', 'oz66', 'oz71', 'oz72', 'oz73', 'oz74', 'oz75', 'oz76', 'oz77', 'oz78', 'oz80', 'oz82', 'oz84', 'oz86', 'oz89', 'oz90', 'oz91', 'oz92', 'oz93', 'oz94', 'oz102', 'oz104', 'oz106', 'oz107', 'oz108', 'oz110', 'oz115', 'oz116', 'oz117', 'oz118', 'oz120', 'oz121', 'oz122', 'oz123', 'oz124', 'oz125', 'oz126', 'oz128', 'oz129', 'oz131', 'oz133', 'oz134', 'oz135', 'oz137', 'oz138', 'oz139', 'oz140', 'oz141', 'oz142', 'oz143', 'oz144', 'oz145', 'oz147', 'oz149', 'oz150', 'oz151', 'oz152', 'oz154', 'oz155', 'oz156', 'oz161', 'oz162', 'oz163', 'oz164', 'oz170', 'oz171', 'oz173', 'oz176', 'oz177', 'oz181', 'oz182', 'oz185', 'oz188', 'oz189', 'oz190', 'oz195', 'oz196', 'oz197', 'oz201', 'oz202', 'oz204', 'oz205', 'oz206', 'oz207', 'oz208', 'oz209', 'oz210', 'oz211', 'oz213', 'oz214', 'oz216', 'oz217', 'oz218', 'oz220', 'oz223', 'oz226', 'oz227', 'oz228', 'oz231', 'oz232', 'oz233', 'oz235', 'oz236', 'oz239', 'oz240', 'oz244', 'oz246', 'oz261'], '335_361288': ['Sex', 'Length', 'Diameter', 'Height', 'Whole_weight', 'Shucked_weight', 'Viscera_weight', 'Shell_weight'], '335_361289': ['Precinct', 'Sector', 'Occurred_hour', 'Occurred_min'], '335_361291': ['vehicle_type', 'direction', 'weekday', 'temp', 'windspeed_max', 'windspeed_avg', 'precipitation', 'dew_point', 'humidity', 'hour', 'dayminute'], '335_361292': ['cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8', 'cat9', 'cat10', 'cat11', 'cat12', 'cat13', 'cat16', 'cat17', 'cat18', 'cat19', 'cat20', 'cat21', 'cat22', 'cat23', 'cat24', 'cat25', 'cat26', 'cat27', 'cat28', 'cat29', 'cat30', 'cat31', 'cat32', 'cat33', 'cat35', 'cat36', 'cat37', 'cat38', 'cat39', 'cat40', 'cat41', 'cat42', 'cat43', 'cat44', 'cat45', 'cat46', 'cat47', 'cat48', 'cat49', 'cat50', 'cat51', 'cat52', 'cat53', 'cat54', 'cat55', 'cat56', 'cat57', 'cat59', 'cat64', 'cat65', 'cat66', 'cat67', 'cat71', 'cat72', 'cat73', 'cat74', 'cat75', 'cat76', 'cat77', 'cat78', 'cat79', 'cat80', 'cat81', 'cat82', 'cat83', 'cat84', 'cat85', 'cat86', 'cat87', 'cat88', 'cat89', 'cat90', 'cat91', 'cat92', 'cat93', 'cat94', 'cat95', 'cat96', 'cat97', 'cat98', 'cat99', 'cat100', 'cat101', 'cat102', 'cat103', 'cat104', 'cat105', 'cat106', 'cat107', 'cat108', 'cat111', 'cat114', 'cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7', 'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14'], '335_361293': ['Month', 'DayofMonth', 'CRSDepTime', 'CRSArrTime', 'Distance'], '335_361294': ['Total_Discharges', 'Average_Covered_Charges', 'Average_Medicare_Payments'], '336_361072': ['lread', 'lwrite', 'scall', 'sread', 'swrite', 'fork', 'exec', 'rchar', 'wchar', 'pgout', 'ppgout', 'pgfree', 'pgscan', 'atch', 'pgin', 'ppgin', 'pflt', 'vflt', 'runqsz', 'freemem', 'freeswap'], '336_361073': ['f5', 'f6', 'f7', 'f8', 'f9', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f26', 'f27', 'f28', 'f29', 'f30', 'f31', 'f32', 'f33'], '336_361074': ['climbRate', 'Sgz', 'p', 'q', 'curRoll', 'absRoll', 'diffClb', 'diffRollRate', 'diffDiffClb', 'SaTime1', 'SaTime2', 'SaTime3', 'SaTime4', 'diffSaTime1', 'diffSaTime3', 'Sa'], '336_361076': ['fixed.acidity', 'volatile.acidity', 'citric.acid', 'residual.sugar', 'chlorides', 'free.sulfur.dioxide', 'total.sulfur.dioxide', 'density', 'pH', 'sulphates', 'alcohol'], '336_361077': ['climbRate', 'Sgz', 'p', 'q', 'curPitch', 'curRoll', 'absRoll', 'diffClb', 'diffRollRate', 'SeTime1', 'SeTime2', 'SeTime3', 'SeTime4', 'SeTime5', 'SeTime6', 'SeTime7', 'SeTime8', 'SeTime9', 'SeTime10', 'SeTime11', 'SeTime12', 'SeTime13', 'SeTime14', 'diffSeTime1', 'diffSeTime3', 'diffSeTime5', 'diffSeTime7', 'diffSeTime9', 'diffSeTime11', 'diffSeTime13', 'alpha', 'Se'], '336_361078': ['median_income', 'housing_median_age', 'total_rooms', 'total_bedrooms', 'population', 'households', 'latitude', 'longitude'], '336_361079': ['P1', 'P5p1', 'P6p2', 'P11p4', 'P14p9', 'P15p1', 'P15p3', 'P16p2', 'P18p2', 'P27p4', 'H2p2', 'H8p2', 'H10p1', 'H13p1', 'H18pA', 'H40p4'], '336_361080': ['carat', 'depth', 'table', 'x', 'y', 'z'], '336_361081': ['area', 'rooms', 'bathroom', 'parking_spaces', 'hoa_(BRL)', 'rent_amount_(BRL)', 'property_tax_(BRL)', 'fire_insurance_(BRL)'], '336_361082': ['month', 'hour', 'temp', 'feel_temp', 'humidity', 'windspeed'], '336_361083': ['passenger_count', 'tolls_amount', 'total_amount', 'lpep_pickup_datetime_day', 'lpep_pickup_datetime_hour', 'lpep_pickup_datetime_minute', 'lpep_dropoff_datetime_day', 'lpep_dropoff_datetime_hour', 'lpep_dropoff_datetime_minute'], '336_361084': ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15', 'date_month', 'date_day'], '336_361085': ['a1', 'a2', 'a3', 'a4', 'a5', 'y2'], '336_361086': ['Total_Discharges', 'Average_Covered_Charges', 'Average_Medicare_Payments'], '336_361087': ['LATITUDE', 'LONGITUDE', 'LND_SQFOOT', 'TOT_LVG_AREA', 'SPEC_FEAT_VAL', 'RAIL_DIST', 'OCEAN_DIST', 'WATER_DIST', 'CNTR_DIST', 'SUBCNTR_DI', 'HWY_DIST', 'age', 'month_sold'], '336_361088': ['mean_atomic_mass', 'wtd_mean_atomic_mass', 'gmean_atomic_mass', 'wtd_gmean_atomic_mass', 'entropy_atomic_mass', 'wtd_entropy_atomic_mass', 'range_atomic_mass', 'wtd_range_atomic_mass', 'std_atomic_mass', 'wtd_std_atomic_mass', 'mean_fie', 'wtd_mean_fie', 'gmean_fie', 'wtd_gmean_fie', 'entropy_fie', 'wtd_entropy_fie', 'range_fie', 'wtd_range_fie', 'std_fie', 'wtd_std_fie', 'mean_atomic_radius', 'wtd_mean_atomic_radius', 'gmean_atomic_radius', 'wtd_gmean_atomic_radius', 'entropy_atomic_radius', 'wtd_entropy_atomic_radius', 'range_atomic_radius', 'wtd_range_atomic_radius', 'std_atomic_radius', 'wtd_std_atomic_radius', 'mean_Density', 'wtd_mean_Density', 'gmean_Density', 'wtd_gmean_Density', 'entropy_Density', 'wtd_entropy_Density', 'range_Density', 'wtd_range_Density', 'std_Density', 'wtd_std_Density', 'mean_ElectronAffinity', 'wtd_mean_ElectronAffinity', 'gmean_ElectronAffinity', 'wtd_gmean_ElectronAffinity', 'entropy_ElectronAffinity', 'wtd_entropy_ElectronAffinity', 'range_ElectronAffinity', 'wtd_range_ElectronAffinity', 'std_ElectronAffinity', 'wtd_std_ElectronAffinity', 'mean_FusionHeat', 'wtd_mean_FusionHeat', 'gmean_FusionHeat', 'wtd_gmean_FusionHeat', 'entropy_FusionHeat', 'wtd_entropy_FusionHeat', 'range_FusionHeat', 'wtd_range_FusionHeat', 'std_FusionHeat', 'wtd_std_FusionHeat', 'mean_ThermalConductivity', 'wtd_mean_ThermalConductivity', 'gmean_ThermalConductivity', 'wtd_gmean_ThermalConductivity', 'entropy_ThermalConductivity', 'wtd_entropy_ThermalConductivity', 'range_ThermalConductivity', 'wtd_range_ThermalConductivity', 'std_ThermalConductivity', 'wtd_std_ThermalConductivity', 'mean_Valence', 'wtd_mean_Valence', 'gmean_Valence', 'wtd_gmean_Valence', 'entropy_Valence', 'wtd_entropy_Valence', 'wtd_range_Valence', 'std_Valence', 'wtd_std_Valence'], '336_361279': ['oz1', 'oz2', 'oz3', 'oz4', 'oz5', 'oz6', 'oz9', 'oz12', 'oz13', 'oz83', 'oz124', 'oz125', 'oz126', 'oz127', 'oz128', 'oz131', 'oz149', 'oz150', 'oz151', 'oz165', 'oz171', 'oz172', 'oz173', 'oz175', 'oz176', 'oz177', 'oz183', 'oz185', 'oz197', 'oz246', 'oz247', 'oz248', 'oz249', 'oz250', 'oz251'], '336_361280': ['Length', 'Diameter', 'Height', 'Whole_weight', 'Shucked_weight', 'Viscera_weight', 'Shell_weight'], '336_361281': ['temp', 'windspeed_max', 'windspeed_avg', 'precipitation', 'dew_point', 'humidity', 'hour', 'dayminute'], '337_361055': ['RevolvingUtilizationOfUnsecuredLines', 'age', 'NumberOfTime30-59DaysPastDueNotWorse', 'DebtRatio', 'MonthlyIncome', 'NumberOfOpenCreditLinesAndLoans', 'NumberOfTimes90DaysLate', 'NumberRealEstateLoansOrLines', 'NumberOfTime60-89DaysPastDueNotWorse', 'NumberOfDependents'], '337_361060': ['date', 'period', 'nswprice', 'nswdemand', 'vicprice', 'vicdemand', 'transfer'], '337_361061': ['X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'X7', 'X8', 'X9', 'X10'], '337_361062': ['f5', 'f6', 'f7', 'f8', 'f9', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24', 'f25', 'f28', 'f29', 'f30', 'f31', 'f32', 'f33'], '337_361063': ['P1', 'P5p1', 'P6p2', 'P11p4', 'P14p9', 'P15p1', 'P15p3', 'P16p2', 'P18p2', 'P27p4', 'H2p2', 'H8p2', 'H10p1', 'H13p1', 'H18pA', 'H40p4'], '337_361065': ['fLength', 'fWidth', 'fSize', 'fConc', 'fConc1', 'fAsym', 'fM3Long', 'fM3Trans', 'fAlpha', 'fDist'], '337_361066': ['V1', 'V6', 'V10', 'V12', 'V13', 'V14', 'V15'], '337_361068': ['ParticleID_0', 'ParticleID_1', 'ParticleID_2', 'ParticleID_3', 'ParticleID_4', 'ParticleID_5', 'ParticleID_6', 'ParticleID_7', 'ParticleID_8', 'ParticleID_9', 'ParticleID_10', 'ParticleID_11', 'ParticleID_12', 'ParticleID_13', 'ParticleID_14', 'ParticleID_15', 'ParticleID_16', 'ParticleID_17', 'ParticleID_18', 'ParticleID_19', 'ParticleID_20', 'ParticleID_21', 'ParticleID_22', 'ParticleID_23', 'ParticleID_24', 'ParticleID_25', 'ParticleID_26', 'ParticleID_27', 'ParticleID_28', 'ParticleID_29', 'ParticleID_30', 'ParticleID_31', 'ParticleID_32', 'ParticleID_33', 'ParticleID_34', 'ParticleID_35', 'ParticleID_36', 'ParticleID_37', 'ParticleID_38', 'ParticleID_39', 'ParticleID_40', 'ParticleID_41', 'ParticleID_42', 'ParticleID_43', 'ParticleID_44', 'ParticleID_45', 'ParticleID_46', 'ParticleID_47', 'ParticleID_48', 'ParticleID_49'], '337_361069': ['lepton_pT', 'lepton_eta', 'lepton_phi', 'missing_energy_magnitude', 'missing_energy_phi', 'jet_1_pt', 'jet_1_eta', 'jet_1_phi', 'jet_2_pt', 'jet_2_eta', 'jet_2_phi', 'jet_3_pt', 'jet_3_eta', 'jet_3_phi', 'jet_4_pt', 'jet_4_eta', 'jet_4_phi', 'm_jj', 'm_jjj', 'm_lv', 'm_jlv', 'm_bb', 'm_wbb', 'm_wwbb'], '337_361070': ['lineNo', 'assgNo', 'prevFixDur', 'firstfixDur', 'firstPassFixDur', 'nextFixDur', 'firstSaccLen', 'lastSaccLen', 'prevFixPos', 'leavingPos', 'regressLen', 'regressDur', 'pupilDiamMax', 'pupilDiamLag', 'timePrtctg', 'titleNo', 'wordNo'], '337_361273': ['time_in_hospital', 'num_lab_procedures', 'num_medications', 'number_outpatient', 'number_emergency', 'number_inpatient', 'number_diagnoses'], '337_361274': ['V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V30', 'V31', 'V35', 'V37', 'V38', 'V41', 'V42', 'V43', 'V44', 'V47', 'V49', 'V51'], '337_361275': ['x1', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10', 'x11', 'x12', 'x13', 'x14', 'x15', 'x16', 'x17', 'x18', 'x19', 'x20', 'x21', 'x22', 'x23'], '337_361276': ['D2', 'D3', 'D5', 'D6', 'D7', 'D8', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'D16', 'D17', 'D18', 'D19', 'D21', 'D25', 'D26', 'D31', 'D32', 'D33', 'D35', 'D36', 'D37', 'D38', 'D39', 'D40', 'D41', 'D42', 'D43', 'D44', 'D45', 'D46', 'D48', 'D49', 'D52', 'D54', 'D55', 'D56', 'D57', 'D58', 'D59', 'D60', 'D61', 'D62', 'D63', 'D64', 'D65', 'D66', 'D68', 'D70', 'D71', 'D73', 'D74', 'D75', 'D78', 'D84', 'D85', 'D86', 'D89', 'D90', 'D91', 'D95', 'D99', 'D101', 'D102', 'D103', 'D104', 'D105', 'D106', 'D107', 'D117', 'D119', 'D123', 'D125', 'D128', 'D131', 'D132', 'D137', 'D140', 'D142', 'D143', 'D144', 'D145', 'D146', 'D149', 'D150', 'D153', 'D154', 'D157', 'D160', 'D172', 'D173', 'D174', 'D175', 'D177', 'D178', 'D180', 'D181', 'D182', 'D183', 'D187', 'D189', 'D191', 'D194', 'D196', 'D197', 'D200', 'D201', 'D202', 'D204', 'D205', 'D207', 'D209', 'D210', 'D211', 'D212', 'D213', 'D215', 'D216', 'D217', 'D218', 'D221', 'D223', 'D224', 'D239', 'D243', 'D249', 'D250', 'D261', 'D312', 'D337', 'D338', 'D374', 'D375', 'D415', 'D417', 'D419', 'D420', 'D422', 'D423', 'D441', 'D442', 'D443', 'D444', 'D446', 'D447', 'D451', 'D454', 'D461', 'D469', 'D471', 'D473', 'D474', 'D477', 'D479', 'D504', 'D505', 'D512', 'D513', 'D514', 'D518', 'D519', 'D520', 'D521', 'D526', 'D546', 'D548', 'D559', 'D560', 'D582', 'D583', 'D586', 'D607', 'D608', 'D621', 'D622', 'D625', 'D649', 'D659', 'D660', 'D661', 'D663', 'D664', 'D672', 'D697', 'D698', 'D700', 'D701', 'D702', 'D741', 'D742', 'D743', 'D749', 'D751', 'D763', 'D764', 'D767', 'D769', 'D770', 'D802', 'D806', 'D809', 'D810', 'D811', 'D814', 'D815', 'D817', 'D889', 'D891', 'D892', 'D897', 'D911', 'D912', 'D926', 'D927', 'D933', 'D935', 'D937', 'D947', 'D948', 'D950', 'D951'], '337_361277': ['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', 'Population', 'AveOccup', 'Latitude', 'Longitude'], '337_361278': ['ExternalRiskEstimate', 'MSinceOldestTradeOpen', 'MSinceMostRecentTradeOpen', 'AverageMInFile', 'NumSatisfactoryTrades', 'NumTrades90Ever2DerogPubRec', 'PercentTradesNeverDelq', 'MSinceMostRecentDelq', 'MaxDelq2PublicRecLast12M', 'NumTotalTrades', 'PercentInstallTrades', 'MSinceMostRecentInqexcl7days', 'NumInqLast6M', 'NumInqLast6Mexcl7days', 'NetFractionRevolvingBurden', 'NetFractionInstallBurden', 'NumRevolvingTradesWBalance', 'NumBank2NatlTradesWHighUtilization', 'PercentTradesWBalance']}\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.07803,"end_time":"2024-05-11T09:28:00.911815","exception":false,"start_time":"2024-05-11T09:28:00.833785","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T17:01:36.572197Z","iopub.execute_input":"2024-05-29T17:01:36.572593Z","iopub.status.idle":"2024-05-29T17:01:36.816369Z","shell.execute_reply.started":"2024-05-29T17:01:36.572561Z","shell.execute_reply":"2024-05-29T17:01:36.815129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation\n\nWe validate the tuned hyperparameters using the same split as in the EDA notebook.","metadata":{"papermill":{"duration":0.005146,"end_time":"2024-05-11T09:28:00.922499","exception":false,"start_time":"2024-05-11T09:28:00.917353","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# Validation with the same split as in the EDA\nkf = StratifiedKFold(shuffle=True, random_state=1)\nscores = defaultdict(lambda: [])\noof = {}\nfor ds_num, identifier in enumerate(datasets.keys()):\n    start_time_task = datetime.datetime.now()\n    scores['identifier'].append(identifier)\n    data = datasets[identifier]\n    \n    # Prepare stratification and train-test split\n    if data.target_type == 'num': # regression\n        assert data.y.shape[1] == 1\n        le = LabelEncoder()\n        stratification = le.fit_transform(pd.qcut(data.y.iloc[:,0], 10, duplicates='drop'))\n    else: # classification\n        stratification = data.y.target\n    idx_tr, idx_va = next(kf.split(data.X, stratification))\n    X_tr, X_va = data.X.iloc[idx_tr], data.X.iloc[idx_va] # DataFrame\n    y_tr, y_va = data.y.iloc[idx_tr, 0], data.y.iloc[idx_va, 0] # Series\n    \n    # Select features\n    X_tr, X_va = X_tr[important_features[identifier]], X_va[important_features[identifier]]\n    \n    # Fit and predict with CatBoost\n    preds = np.zeros_like(y_va, dtype=float)\n    for random_state in range(n_repeats):\n        if data.target_type == 'num': # regression\n            model = catboost.CatBoostRegressor(**cb_params[identifier][0], random_state=random_state,\n                                               cat_features=X_tr.columns[X_tr.dtypes == 'category'].values)\n            model.fit(X_tr, y_tr)\n            y_va_pred = model.predict(X_va)\n            score1 = mean_squared_error(y_va, y_va_pred)\n        else: # classification\n            model = catboost.CatBoostClassifier(**cb_params[identifier][0], random_state=random_state,\n                                                cat_features=X_tr.columns[X_tr.dtypes == 'category'].values)\n            model.fit(X_tr, y_tr)\n            y_va_pred = model.predict_proba(X_va)[:, 1]\n            score1 = log_loss(y_va, y_va_pred)\n        preds += y_va_pred\n    preds /= n_repeats\n    if data.target_type == 'num': # regression\n        score = mean_squared_error(y_va, preds)\n    else: # classification\n        score = log_loss(y_va, preds)\n    duration_task = datetime.datetime.now() - start_time_task\n    print(f\"{ds_num:2} {identifier} {data.target_type}: cb {duration_task.seconds:5} s   {score1:12.4e} {score:12.4e} \")\n    scores['cb1'].append(score1)\n    scores['cb'].append(score)\n    oof[identifier] = preds.astype(np.float32)\nwith open('oof1.pickle', 'wb') as f:\n    pickle.dump(oof, f)\n","metadata":{"_kg_hide-output":true,"papermill":{"duration":709.9798,"end_time":"2024-05-11T09:39:50.908159","exception":false,"start_time":"2024-05-11T09:28:00.928359","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T17:01:36.817880Z","iopub.execute_input":"2024-05-29T17:01:36.818246Z","iopub.status.idle":"2024-05-29T20:27:06.334622Z","shell.execute_reply.started":"2024-05-29T17:01:36.818202Z","shell.execute_reply":"2024-05-29T20:27:06.333447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can now compare the CatBoost scores to the scores of the other three notebooks. The second-to-last column (`winner`) indicates which of the four is the best model.","metadata":{"papermill":{"duration":0.011015,"end_time":"2024-05-11T09:39:50.930211","exception":false,"start_time":"2024-05-11T09:39:50.919196","status":"completed"},"tags":[]}},{"cell_type":"code","source":"cv1_df = pd.DataFrame(scores).set_index('identifier')\nwith open('/kaggle/input/tdb2-only-xgboost/cv1.pickle', 'rb') as f:\n    cv1_df['xgb'] = pickle.load(f)['xgb'].values\nwith open('/kaggle/input/tdb2-only-lightgbm/cv1.pickle', 'rb') as f:\n    cv1_df['lightgbm'] = pickle.load(f)['lightgbm'].values\nwith open('/kaggle/input/tdb2-only-hgb/cv1.pickle', 'rb') as f:\n    cv1_df['hgb'] = pickle.load(f)['hgb'].values\n\nresult_df = cv1_df.copy().reset_index()\nresult_df['winner'] = cv1_df.columns[np.argmin(cv1_df.fillna(1e12), axis=1)]\nresult_df['best'] = cv1_df.fillna(1e12).min(axis=1).values\n\nwith open('cv1.pickle', 'wb') as f:\n    pickle.dump(result_df, f)\nresult_df.style.format(precision=5).map(lambda s: 'background-color:lightgreen;' if str(s).startswith('c') else None)\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.126026,"end_time":"2024-05-11T09:39:51.067511","exception":false,"start_time":"2024-05-11T09:39:50.941485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T20:27:06.336111Z","iopub.execute_input":"2024-05-29T20:27:06.336486Z","iopub.status.idle":"2024-05-29T20:27:06.658587Z","shell.execute_reply.started":"2024-05-29T20:27:06.336455Z","shell.execute_reply":"2024-05-29T20:27:06.657445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nWe retrain the models on the full datasets and concatenate the predictions to a submission file.","metadata":{"papermill":{"duration":0.011694,"end_time":"2024-05-11T09:39:51.091042","exception":false,"start_time":"2024-05-11T09:39:51.079348","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nsubmission = []\nfor ds_num, identifier in tqdm(list(enumerate(datasets.keys()))):\n    data = datasets[identifier]\n    X_tr = data.X # DataFrame\n    y_tr = data.y.iloc[:, 0] # Series\n    \n    # Select features\n    X_tr, X_te = X_tr[important_features[identifier]], data.X_te[important_features[identifier]]\n    \n    # Fit and predict with CatBoost\n    preds = np.zeros(len(data.X_te), dtype=float)\n    for random_state in range(n_repeats):\n        if data.target_type == 'num': # regression\n            model = catboost.CatBoostRegressor(**cb_params[identifier][0], random_state=random_state,\n                                               cat_features=X_tr.columns[X_tr.dtypes == 'category'].values)\n            model.fit(X_tr, y_tr)\n            y_pred = model.predict(X_te)\n        else: # classification\n            model = catboost.CatBoostClassifier(**cb_params[identifier][0], random_state=random_state,\n                                                cat_features=X_tr.columns[X_tr.dtypes == 'category'].values)\n            model.fit(X_tr, y_tr)\n            y_pred = model.predict_proba(X_te)[:, 1]\n        preds += y_pred\n    preds /= n_repeats\n\n    # Add index for submission\n    preds = pd.Series(preds,\n                      index=pd.Index(identifier + '_' + data.X_te.index.astype(str), name='ID'),\n                      name='target')\n    submission.append(preds)\n    \nprint()\nsubmission = pd.concat(submission)\nsubmission.to_csv('submission.csv', index=True)\n!head submission.csv","metadata":{"papermill":{"duration":965.150738,"end_time":"2024-05-11T09:55:56.254007","exception":false,"start_time":"2024-05-11T09:39:51.103269","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-29T20:27:06.659947Z","iopub.execute_input":"2024-05-29T20:27:06.660405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.017337,"end_time":"2024-05-11T09:55:56.289212","exception":false,"start_time":"2024-05-11T09:55:56.271875","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}