{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-04T02:08:16.042283Z","iopub.execute_input":"2024-10-04T02:08:16.042932Z","iopub.status.idle":"2024-10-04T02:08:17.214620Z","shell.execute_reply.started":"2024-10-04T02:08:16.042889Z","shell.execute_reply":"2024-10-04T02:08:17.213632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from prettytable import PrettyTable\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder,MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nimport xgboost as xgb\nimport lightgbm as lgb\nimport optuna\nimport logging\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:10:01.307604Z","iopub.execute_input":"2024-10-04T02:10:01.308206Z","iopub.status.idle":"2024-10-04T02:10:05.821591Z","shell.execute_reply.started":"2024-10-04T02:10:01.308167Z","shell.execute_reply":"2024-10-04T02:10:05.820329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"/kaggle/input/brist1d/train.csv\",low_memory=False)\ntest=pd.read_csv(\"/kaggle/input/brist1d/test.csv\",low_memory=False)\nsubmission_df = pd.read_csv('/kaggle/input/brist1d/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:10:05.823405Z","iopub.execute_input":"2024-10-04T02:10:05.824000Z","iopub.status.idle":"2024-10-04T02:10:27.888518Z","shell.execute_reply.started":"2024-10-04T02:10:05.823947Z","shell.execute_reply":"2024-10-04T02:10:27.887547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data exploration","metadata":{}},{"cell_type":"markdown","source":"View the missing ratio of all feature columns","metadata":{}},{"cell_type":"code","source":"table = PrettyTable()\ntarget = 'bg+1:00'\ntable.field_names = ['Feature', 'Data Type', 'Train Missing %', 'Test Missing %']\nfor column in train.columns:\n    data_type = str(train[column].dtype)\n    non_null_count_train= np.round(100-train[column].count()/train.shape[0]*100,1)\n    if column!=target:\n        non_null_count_test = np.round(100-test[column].count()/test.shape[0]*100,1)\n    else:\n        non_null_count_test=\"NA\"\n    table.add_row([column, data_type, non_null_count_train,non_null_count_test])\nprint(table)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:12:13.052002Z","iopub.execute_input":"2024-10-04T02:12:13.052441Z","iopub.status.idle":"2024-10-04T02:12:13.871281Z","shell.execute_reply.started":"2024-10-04T02:12:13.052404Z","shell.execute_reply":"2024-10-04T02:12:13.870074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:22:41.908533Z","iopub.execute_input":"2024-10-04T08:22:41.909443Z","iopub.status.idle":"2024-10-04T08:22:41.914254Z","shell.execute_reply.started":"2024-10-04T08:22:41.909397Z","shell.execute_reply":"2024-10-04T08:22:41.913101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_cols = [col for col in train.columns if train[col].dtype in ['float64', 'int64']]\ntrain_numeric = train[numeric_cols]\nprint(train_numeric.describe())","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:12:48.555991Z","iopub.execute_input":"2024-10-04T02:12:48.556547Z","iopub.status.idle":"2024-10-04T02:12:52.392211Z","shell.execute_reply.started":"2024-10-04T02:12:48.556505Z","shell.execute_reply":"2024-10-04T02:12:52.391016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{}},{"cell_type":"markdown","source":"The time format is %H:%M:%S, indicating the format of hour: minute: second","metadata":{}},{"cell_type":"code","source":"train['time'] = pd.to_datetime(train['time'], format='%H:%M:%S')\ntest['time'] = pd.to_datetime(test['time'], format='%H:%M:%S')","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:12:57.530407Z","iopub.execute_input":"2024-10-04T02:12:57.531181Z","iopub.status.idle":"2024-10-04T02:12:57.579742Z","shell.execute_reply.started":"2024-10-04T02:12:57.531136Z","shell.execute_reply":"2024-10-04T02:12:57.578868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['hour'] = train['time'].dt.hour\ntrain['minute'] = train['time'].dt.minute\ntest['hour'] = test['time'].dt.hour\ntest['minute'] = test['time'].dt.minute","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:05.266773Z","iopub.execute_input":"2024-10-04T02:13:05.267157Z","iopub.status.idle":"2024-10-04T02:13:05.286029Z","shell.execute_reply.started":"2024-10-04T02:13:05.267122Z","shell.execute_reply":"2024-10-04T02:13:05.284907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('time', axis=1, inplace=True)\ntest.drop('time', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:06.591857Z","iopub.execute_input":"2024-10-04T02:13:06.592252Z","iopub.status.idle":"2024-10-04T02:13:06.931384Z","shell.execute_reply.started":"2024-10-04T02:13:06.592217Z","shell.execute_reply":"2024-10-04T02:13:06.930309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Select columns of data type float64 or int64 from the training set train and convert their column names to a list stored in numerical_cols. Remove the column named 'bg+1:00' from this list.\n\nFilter the column name containing the string 'activity' from the column name of the training set and store it in categorical_cols.","metadata":{}},{"cell_type":"code","source":"numerical_cols = train.select_dtypes(include=['float64', 'int64']).columns.tolist()\nnumerical_cols.remove('bg+1:00')  \ncategorical_cols = [col for col in train.columns if 'activity' in col]","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:07.892829Z","iopub.execute_input":"2024-10-04T02:13:07.893615Z","iopub.status.idle":"2024-10-04T02:13:08.114093Z","shell.execute_reply.started":"2024-10-04T02:13:07.893576Z","shell.execute_reply":"2024-10-04T02:13:08.113104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The NaN of the corresponding column in the training set train and test set test is populated with 'None'","metadata":{}},{"cell_type":"code","source":"for col in categorical_cols:\n    train[col] = train[col].fillna('None')\n    test[col] = test[col].fillna('None')","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:09.967899Z","iopub.execute_input":"2024-10-04T02:13:09.968855Z","iopub.status.idle":"2024-10-04T02:13:10.894979Z","shell.execute_reply.started":"2024-10-04T02:13:09.968810Z","shell.execute_reply":"2024-10-04T02:13:10.894109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering, using MinMaxScaler","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\nfor col in categorical_cols:\n    combined_data = pd.concat([train[col], test[col]], axis=0)\n    le.fit(combined_data)\n    train[col] = le.transform(train[col])\n    test[col] = le.transform(test[col])","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:11.295791Z","iopub.execute_input":"2024-10-04T02:13:11.296741Z","iopub.status.idle":"2024-10-04T02:13:14.602818Z","shell.execute_reply.started":"2024-10-04T02:13:11.296699Z","shell.execute_reply":"2024-10-04T02:13:14.601814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()\ntrain[numerical_cols] = scaler.fit_transform(train[numerical_cols])\ntest[numerical_cols] = scaler.transform(test[numerical_cols])","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:14.604765Z","iopub.execute_input":"2024-10-04T02:13:14.605106Z","iopub.status.idle":"2024-10-04T02:13:15.798238Z","shell.execute_reply.started":"2024-10-04T02:13:14.605070Z","shell.execute_reply":"2024-10-04T02:13:15.796935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model building","metadata":{}},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"X = train.drop(['id', 'p_num', 'bg+1:00'], axis=1)\ny= train['bg+1:00']","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:15.799509Z","iopub.execute_input":"2024-10-04T02:13:15.799869Z","iopub.status.idle":"2024-10-04T02:13:16.484304Z","shell.execute_reply.started":"2024-10-04T02:13:15.799834Z","shell.execute_reply":"2024-10-04T02:13:16.483176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:16.486463Z","iopub.execute_input":"2024-10-04T02:13:16.486823Z","iopub.status.idle":"2024-10-04T02:13:17.959631Z","shell.execute_reply.started":"2024-10-04T02:13:16.486789Z","shell.execute_reply":"2024-10-04T02:13:17.958669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create the DMatrix data structure","metadata":{}},{"cell_type":"code","source":"dtrain = xgb.DMatrix(X_train, label=y_train)\ndval = xgb.DMatrix(X_val, label=y_val)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:13:17.960769Z","iopub.execute_input":"2024-10-04T02:13:17.961137Z","iopub.status.idle":"2024-10-04T02:13:19.796260Z","shell.execute_reply.started":"2024-10-04T02:13:17.961099Z","shell.execute_reply":"2024-10-04T02:13:19.795364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'booster': 'gbtree',\n    'learning_rate': 0.1,\n    'max_depth': 6,\n    'device': 'cuda',\n    'verbosity': 0,\n    'gamma': 0.1,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 0.1,\n    'reg_lambda': 0.1\n}\nnum_rounds = 100\nmodel = xgb.train(params, dtrain, num_rounds, evals=[(dval, 'eval')],verbose_eval=False)\ny_pred = model.predict(dval)\nmse = mean_squared_error(y_val, y_pred)\nrmse = np.sqrt(mse)\nprint(f'Root Mean Squared Error: {rmse}')","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:15:53.182433Z","iopub.execute_input":"2024-10-04T02:15:53.183223Z","iopub.status.idle":"2024-10-04T02:15:55.571426Z","shell.execute_reply.started":"2024-10-04T02:15:53.183183Z","shell.execute_reply":"2024-10-04T02:15:55.570133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameter tuning with optune","metadata":{}},{"cell_type":"markdown","source":"Perform five hyperparameter tuning and store the results in the list","metadata":{}},{"cell_type":"code","source":"logging.getLogger('optuna').setLevel(logging.WARNING)\n\ndef objective(trial):\n    learning_rate = trial.suggest_float('learning_rate', 0.01, 0.2)\n    max_depth = trial.suggest_int('max_depth', 3, 10)\n    gamma = trial.suggest_float('gamma', 0, 0.5)\n    subsample = trial.suggest_float('subsample', 0.5, 1)\n    colsample_bytree = trial.suggest_float('colsample_bytree', 0.5, 1)\n    reg_alpha = trial.suggest_float('reg_alpha', 0, 1)\n    reg_lambda = trial.suggest_float('reg_lambda', 0, 1)\n\n    params = {\n        'objective': 'reg:squarederror',\n        'eval_metric': 'rmse',\n        'booster': 'gbtree',\n        'learning_rate': learning_rate,\n        'max_depth': max_depth,\n        'gamma': gamma,\n        'subsample': subsample,\n        'colsample_bytree': colsample_bytree,\n        'reg_alpha': reg_alpha,\n        'reg_lambda': reg_lambda,\n        'device': 'cuda',\n        'verbosity': 0\n    }\n    num_rounds = 100\n    model = xgb.train(params, dtrain, num_rounds, evals=[(dval, 'eval')],verbose_eval=False)\n    y_pred = model.predict(dval)\n    mse = mean_squared_error(y_val, y_pred)\n    return mse\n\nresults = []\nfor _ in range(5):\n    study = optuna.create_study(direction='minimize')\n    study.optimize(objective, n_trials=50)\n    results.append((study.best_params, study.best_value))","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:16:44.499860Z","iopub.execute_input":"2024-10-04T02:16:44.500287Z","iopub.status.idle":"2024-10-04T02:29:40.171745Z","shell.execute_reply.started":"2024-10-04T02:16:44.500247Z","shell.execute_reply":"2024-10-04T02:29:40.170618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Convert the list to DataFrame and observe the changes of the five parameter adjustments, we can see that learning_rate varies from 0.1 to 0.2. In the next interval of hyperparameter optimization, we can set the interval to 0.1 to 0.2. In max_depth, we can see that there is no change, but the interval is set too small. The interval should be increased, as should the other parameters, and the interval should be further reduced or approximated by observing changes.","metadata":{}},{"cell_type":"code","source":"columns = ['learning_rate', 'max_depth', 'gamma', 'subsample', 'colsample_bytree', 'reg_alpha', 'reg_lambda', 'result']\ndf_p= []\nfor item in results:\n    params_dict = item[0]\n    result = item[1]\n    row_d = [params_dict.get(key, None) for key in columns[:-1]] + [result]\n    df_p.append(row_d)\nParameters_1 = pd.DataFrame(df_p, columns=columns)\nParameters_1","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:41:22.540338Z","iopub.execute_input":"2024-10-04T02:41:22.540769Z","iopub.status.idle":"2024-10-04T02:41:22.558587Z","shell.execute_reply.started":"2024-10-04T02:41:22.540731Z","shell.execute_reply":"2024-10-04T02:41:22.557249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hyperparameter optimization is performed according to the new interval","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    learning_rate = trial.suggest_float('learning_rate', 0.1, 0.2)\n    max_depth = trial.suggest_int('max_depth', 10, 31)\n    gamma = trial.suggest_float('gamma', 0.1, 0.5)\n    subsample = trial.suggest_float('subsample', 0.7, 1)\n    colsample_bytree = trial.suggest_float('colsample_bytree', 0.5, 0.95)\n    reg_alpha = trial.suggest_float('reg_alpha', 0.1, 0.8)\n    reg_lambda = trial.suggest_float('reg_lambda', 0, 0.932)\n\n    params = {\n        'objective': 'reg:squarederror',\n        'eval_metric': 'rmse',\n        'booster': 'gbtree',\n        'learning_rate': learning_rate,\n        'max_depth': max_depth,\n        'gamma': gamma,\n        'subsample': subsample,\n        'colsample_bytree': colsample_bytree,\n        'reg_alpha': reg_alpha,\n        'reg_lambda': reg_lambda,\n        'device': 'cuda',\n        'verbosity': 0\n    }\n    num_rounds = 100\n    model = xgb.train(params, dtrain, num_rounds, evals=[(dval, 'eval')],verbose_eval=False)\n    y_pred = model.predict(dval)\n    mse = mean_squared_error(y_val, y_pred)\n    return mse\n\n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\nprint(\"Best parameters:\", study.best_params)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T02:57:40.385202Z","iopub.execute_input":"2024-10-04T02:57:40.386103Z","iopub.status.idle":"2024-10-04T03:14:23.638708Z","shell.execute_reply.started":"2024-10-04T02:57:40.386054Z","shell.execute_reply":"2024-10-04T03:14:23.637661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use new parameters","metadata":{}},{"cell_type":"code","source":"params_1 = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'booster': 'gbtree',\n    'learning_rate': 0.12179027008335803,\n    'max_depth': 17,\n    'device': 'cuda',\n    'verbosity': 0,\n    'gamma': 0.3522778611442141,\n    'subsample': 0.7579337214351011,\n    'colsample_bytree': 0.7253353621476007,\n    'reg_alpha': 0.11563451629533103,\n    'reg_lambda': 0.7284436234535214\n}\nnum_rounds = 100\nmodel_xgb = xgb.train(params_1, dtrain, num_rounds, evals=[(dval, 'eval')],verbose_eval=False)\ny_pred = model_xgb.predict(dval)\nmse = mean_squared_error(y_val, y_pred)\nrmse = np.sqrt(mse)\nprint(f'Root Mean Squared Error: {rmse}')","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:20:36.468118Z","iopub.execute_input":"2024-10-04T03:20:36.468858Z","iopub.status.idle":"2024-10-04T03:20:56.261778Z","shell.execute_reply.started":"2024-10-04T03:20:36.468818Z","shell.execute_reply":"2024-10-04T03:20:56.260733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM","metadata":{}},{"cell_type":"markdown","source":"Before you start, make a copy of the data and process the column names","metadata":{}},{"cell_type":"code","source":"train_lgbm = train.copy()","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:38:57.450902Z","iopub.execute_input":"2024-10-04T03:38:57.451370Z","iopub.status.idle":"2024-10-04T03:38:58.017087Z","shell.execute_reply.started":"2024-10-04T03:38:57.451306Z","shell.execute_reply":"2024-10-04T03:38:58.016261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n    \ntrain_lgbm= train_lgbm.rename(columns={'bg+1:00': 'bg+1hour'}, inplace=False)\ndef clean_column_names(train_lgbm):\n    train_lgbm.columns = [re.sub(r'[^\\w\\s]', '_', col) for col in train_lgbm.columns]\n    return train_lgbm\ntrain_lgbm = clean_column_names (train_lgbm)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:42:22.838946Z","iopub.execute_input":"2024-10-04T03:42:22.839657Z","iopub.status.idle":"2024-10-04T03:42:23.052227Z","shell.execute_reply.started":"2024-10-04T03:42:22.839611Z","shell.execute_reply":"2024-10-04T03:42:23.051189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_lgbm.drop(['id', 'p_num', 'bg_1hour'], axis=1)\ny= train_lgbm['bg_1hour']","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:43:44.961780Z","iopub.execute_input":"2024-10-04T03:43:44.962565Z","iopub.status.idle":"2024-10-04T03:43:45.169046Z","shell.execute_reply.started":"2024-10-04T03:43:44.962523Z","shell.execute_reply":"2024-10-04T03:43:45.168172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:43:54.686054Z","iopub.execute_input":"2024-10-04T03:43:54.686439Z","iopub.status.idle":"2024-10-04T03:43:55.375170Z","shell.execute_reply.started":"2024-10-04T03:43:54.686405Z","shell.execute_reply":"2024-10-04T03:43:55.374313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use LightGBM's lgb.Dataset efficient data structure to store data for use in model training","metadata":{}},{"cell_type":"code","source":"lgb_train = lgb.Dataset(X_train, y_train)\nlgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:43:59.026290Z","iopub.execute_input":"2024-10-04T03:43:59.026700Z","iopub.status.idle":"2024-10-04T03:43:59.033468Z","shell.execute_reply.started":"2024-10-04T03:43:59.026665Z","shell.execute_reply":"2024-10-04T03:43:59.032485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_lg = {\n    'objective': 'regression',\n    'metric': 'gbdt',\n    'verbose': 0,\n    'device': 'gpu',\n    'gpu_platform_id':0, \n    'gpu_device_id':0\n}\n\nmodel = lgb.train(params_lg, lgb_train, valid_sets=[lgb_eval], num_boost_round=100)\ny_pred = model.predict(X_val)\nrmse = np.sqrt(mean_squared_error(y_val, y_pred))\nprint(f\"Root Mean Squared Error: {rmse}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-04T03:57:01.925010Z","iopub.execute_input":"2024-10-04T03:57:01.925735Z","iopub.status.idle":"2024-10-04T03:57:07.800167Z","shell.execute_reply.started":"2024-10-04T03:57:01.925695Z","shell.execute_reply":"2024-10-04T03:57:07.799345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use optune for hyperparameter tuning, perform 5 times, store the results in a list, observe the changes in each parameter, and determine the approximate interval","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    num_leaves = trial.suggest_int('num_leaves', 21, 61)\n    max_depth = trial.suggest_int('max_depth', -1, 15)\n    learning_rate = trial.suggest_float('learning_rate', 0.01, 0.5, log=True)\n    subsample = trial.suggest_float('subsample', 0.2, 1.0)\n    colsample_bytree = trial.suggest_float('colsample_bytree', 0.4, 1.0)\n    reg_alpha = trial.suggest_float('reg_alpha', 0.01, 0.2, log=True)\n    reg_lambda = trial.suggest_float('reg_lambda', 0.01, 0.2, log=True)\n\n    params_lg = {\n        'objective': 'regression',\n        'metric': 'gbdt',\n        'verbose': -1,\n        'device': 'gpu',\n        'gpu_platform_id': 0,\n        'gpu_device_id': 0,\n        'num_leaves': num_leaves,\n        'max_depth': max_depth,\n        'learning_rate': learning_rate,\n        'subsample': subsample,\n        'colsample_bytree': colsample_bytree,\n        'reg_alpha': reg_alpha,\n        'reg_lambda': reg_lambda\n    }\n    num_rounds = 100\n    model = lgb.train(params_lg, lgb_train, valid_sets=[lgb_eval], num_boost_round=num_rounds)\n    y_pred = model.predict(X_val)\n    rmse = np.sqrt(mean_squared_error(y_val, y_pred))\n    return rmse\n\nresults_lg = []\nstudy = optuna.create_study(direction='minimize')\n\nfor _ in range(5):\n    study.optimize(objective, n_trials=50)\n    results_lg.append((study.best_params, study.best_value))","metadata":{"execution":{"iopub.status.busy":"2024-10-04T04:25:49.552743Z","iopub.execute_input":"2024-10-04T04:25:49.553661Z","iopub.status.idle":"2024-10-04T04:56:21.603125Z","shell.execute_reply.started":"2024-10-04T04:25:49.553622Z","shell.execute_reply":"2024-10-04T04:56:21.601982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['num_leaves', 'max_depth', 'learning_rate', 'subsample', 'colsample_bytree', 'reg_alpha', 'reg_lambda']\ndf_lg = []\nfor item in results_lg:\n    params_dict = item[0]\n    row_d = [params_dict.get(key, None) for key in columns]\n    df_lg.append(row_d)\nParameters_lg = pd.DataFrame(df_lg, columns=columns)\nParameters_lg","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:19:45.759071Z","iopub.execute_input":"2024-10-04T05:19:45.760153Z","iopub.status.idle":"2024-10-04T05:19:45.781950Z","shell.execute_reply.started":"2024-10-04T05:19:45.760094Z","shell.execute_reply":"2024-10-04T05:19:45.780762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"According to the results of five times, a new interval is determined and the hyperparameter tuning is performed again","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    num_leaves = trial.suggest_int('num_leaves', 58, 70)\n    max_depth = trial.suggest_int('max_depth', -1, 20)\n    learning_rate = trial.suggest_float('learning_rate', 0.3, 0.4, log=True)\n    subsample = trial.suggest_float('subsample', 0.38, 0.95)\n    colsample_bytree = trial.suggest_float('colsample_bytree', 0.4, 0.66)\n    reg_alpha = trial.suggest_float('reg_alpha', 0.05, 0.17, log=True)\n    reg_lambda = trial.suggest_float('reg_lambda', 0.01, 0.19, log=True)\n\n    params_lg = {\n        'objective': 'regression',\n        'metric': 'gbdt',\n        'verbose': -1,\n        'device': 'gpu',\n        'gpu_platform_id': 0,\n        'gpu_device_id': 0,\n        'num_leaves': num_leaves,\n        'max_depth': max_depth,\n        'learning_rate': learning_rate,\n        'subsample': subsample,\n        'colsample_bytree': colsample_bytree,\n        'reg_alpha': reg_alpha,\n        'reg_lambda': reg_lambda\n    }\n    num_rounds = 100\n    model = lgb.train(params_lg, lgb_train, valid_sets=[lgb_eval], num_boost_round=num_rounds)\n    y_pred = model.predict(X_val)\n    rmse = np.sqrt(mean_squared_error(y_val, y_pred))\n    return rmse\n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\nprint(\"Best parameters:\", study.best_params)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:24:27.863905Z","iopub.execute_input":"2024-10-04T05:24:27.864792Z","iopub.status.idle":"2024-10-04T05:28:57.941595Z","shell.execute_reply.started":"2024-10-04T05:24:27.864750Z","shell.execute_reply":"2024-10-04T05:28:57.939203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train the model with the new parameters","metadata":{}},{"cell_type":"code","source":"params_lg = {\n    'objective': 'regression',\n    'metric': 'gbdt',\n    'verbose': 0,\n    'device': 'gpu',\n    'gpu_platform_id':0, \n    'gpu_device_id':0,\n    'num_leaves': 70,\n    'max_depth': -1,\n    'learning_rate': 0.3790851917910561,\n    'subsample': 0.7419514798024587,\n    'colsample_bytree': 0.47028863655698244,\n    'reg_alpha': 0.1527856365022262,\n    'reg_lambda': 0.09271477244213412\n}\n\nmodel_lgb = lgb.train(params_lg, lgb_train, valid_sets=[lgb_eval], num_boost_round=100)\ny_pred = model_lgb.predict(X_val)\nrmse = np.sqrt(mean_squared_error(y_val, y_pred))\nprint(f\"Root Mean Squared Error: {rmse}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:32:38.531694Z","iopub.execute_input":"2024-10-04T05:32:38.532718Z","iopub.status.idle":"2024-10-04T05:32:46.451979Z","shell.execute_reply.started":"2024-10-04T05:32:38.532674Z","shell.execute_reply":"2024-10-04T05:32:46.449847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost","metadata":{}},{"cell_type":"code","source":"train_cat = train.copy()","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:34:10.819489Z","iopub.execute_input":"2024-10-04T05:34:10.820455Z","iopub.status.idle":"2024-10-04T05:34:11.335865Z","shell.execute_reply.started":"2024-10-04T05:34:10.820413Z","shell.execute_reply":"2024-10-04T05:34:11.334835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_cat.drop(['id', 'p_num', 'bg+1:00'], axis=1)\ny= train_cat['bg+1:00']","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:35:46.352793Z","iopub.execute_input":"2024-10-04T05:35:46.353212Z","iopub.status.idle":"2024-10-04T05:35:46.557298Z","shell.execute_reply.started":"2024-10-04T05:35:46.353176Z","shell.execute_reply":"2024-10-04T05:35:46.556399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:36:06.605402Z","iopub.execute_input":"2024-10-04T05:36:06.606268Z","iopub.status.idle":"2024-10-04T05:36:07.167951Z","shell.execute_reply.started":"2024-10-04T05:36:06.606229Z","shell.execute_reply":"2024-10-04T05:36:07.167127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostRegressor\nmodel = CatBoostRegressor(task_type=\"GPU\")\nmodel.fit(X_train, y_train, eval_set=(X_val, y_val), verbose=False)\ny_pred = model.predict(X_val)\nrmse = np.sqrt(mean_squared_error(y_val, y_pred))\nprint(f\"Root Mean Squared Error: {rmse}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:39:27.580081Z","iopub.execute_input":"2024-10-04T05:39:27.580766Z","iopub.status.idle":"2024-10-04T05:39:38.040631Z","shell.execute_reply.started":"2024-10-04T05:39:27.580724Z","shell.execute_reply":"2024-10-04T05:39:38.039555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\ndef objective(params):\n    model = CatBoostRegressor(\n        task_type=\"GPU\",\n        learning_rate=params['learning_rate'],\n        depth=params['depth'],\n        l2_leaf_reg=params['l2_leaf_reg'],\n        iterations=params['iterations']\n    )\n    model.fit(X_train, y_train, eval_set=(X_val, y_val), verbose=False)\n    y_pred = model.predict(X_val)\n    rmse = np.sqrt(mean_squared_error(y_val, y_pred))\n    return {'loss': rmse, 'status': STATUS_OK}\nspace = {\n    'learning_rate': hp.uniform('learning_rate', 0.01, 0.2),\n    'depth': hp.randint('depth', 3, 10),\n    'l2_leaf_reg': hp.uniform('l2_leaf_reg', 1, 20),\n    'iterations': hp.randint('iterations', 100, 1000)\n}\n\nresults_cat = []\nfor _ in range(5):\n    trials = Trials()\n    best = fmin(objective, space, algo=tpe.suggest, max_evals=150, trials=trials)\n    results_cat.append(best)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T05:56:18.778227Z","iopub.execute_input":"2024-10-04T05:56:18.779172Z","iopub.status.idle":"2024-10-04T08:05:21.071031Z","shell.execute_reply.started":"2024-10-04T05:56:18.779129Z","shell.execute_reply":"2024-10-04T08:05:21.069924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Parameters_cat = pd.DataFrame(results_cat)\nParameters_cat","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:12:01.138719Z","iopub.execute_input":"2024-10-04T08:12:01.139122Z","iopub.status.idle":"2024-10-04T08:12:01.151588Z","shell.execute_reply.started":"2024-10-04T08:12:01.139083Z","shell.execute_reply":"2024-10-04T08:12:01.150597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_cat","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:15:48.769573Z","iopub.execute_input":"2024-10-04T08:15:48.770259Z","iopub.status.idle":"2024-10-04T08:15:48.777370Z","shell.execute_reply.started":"2024-10-04T08:15:48.770221Z","shell.execute_reply":"2024-10-04T08:15:48.776434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cat = CatBoostRegressor(\n        task_type=\"GPU\",\n        learning_rate=0.18865165637295442,\n        depth=9,\n        l2_leaf_reg=1.9773127596229534,\n        iterations=980\n    )\nmodel_cat.fit(X_train, y_train, eval_set=(X_val, y_val), verbose=False)\ny_pred = model_cat.predict(X_val)\nrmse = np.sqrt(mean_squared_error(y_val, y_pred))\nprint(f\"Root Mean Squared Error: {rmse}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:20:12.724966Z","iopub.execute_input":"2024-10-04T08:20:12.725417Z","iopub.status.idle":"2024-10-04T08:20:36.230969Z","shell.execute_reply.started":"2024-10-04T08:20:12.725373Z","shell.execute_reply":"2024-10-04T08:20:36.229991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate submission","metadata":{}},{"cell_type":"markdown","source":"Before we get started, we need to do a little processing on test so that we can use the model to make predictions","metadata":{}},{"cell_type":"code","source":"test_1 = test.drop(['id', 'p_num'], axis=1)\ntest_dmatrix = xgb.DMatrix(test_1)\npredictions_1 = model_xgb.predict(test_dmatrix)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:37:26.872924Z","iopub.execute_input":"2024-10-04T08:37:26.873712Z","iopub.status.idle":"2024-10-04T08:37:27.048234Z","shell.execute_reply.started":"2024-10-04T08:37:26.873671Z","shell.execute_reply":"2024-10-04T08:37:27.047455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['bg+1:00'] = predictions_1","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:38:06.003172Z","iopub.execute_input":"2024-10-04T08:38:06.003579Z","iopub.status.idle":"2024-10-04T08:38:06.008353Z","shell.execute_reply.started":"2024-10-04T08:38:06.003541Z","shell.execute_reply":"2024-10-04T08:38:06.007420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-04T08:38:18.511124Z","iopub.execute_input":"2024-10-04T08:38:18.511839Z","iopub.status.idle":"2024-10-04T08:38:18.527529Z","shell.execute_reply.started":"2024-10-04T08:38:18.511797Z","shell.execute_reply":"2024-10-04T08:38:18.526500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}