{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Data Loading & Exploration","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import MinMaxScaler\nimport joblib  \nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.impute import SimpleImputer\nimport time  # To measure training time\n\n# time - time of day in the format HH:MM:SS (every 15 mins)\n# p_num - participant number, 9 patients data in train, 15 patients data in test (some patients in train not in test)\n\ntrain = pd.read_csv('/kaggle/input/brist1d/train.csv',low_memory=False)\ntest = pd.read_csv('/kaggle/input/brist1d/test.csv',low_memory=False)\nprint(\"Train:\")\nprint(train[\"p_num\"].unique())\ndisplay(train.head())\ndisplay(train.shape)\n\nprint(\"Test:\")\nprint(test[\"p_num\"].unique())\ndisplay(test.head())\ndisplay(test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:25:41.387488Z","iopub.execute_input":"2024-10-01T08:25:41.388933Z","iopub.status.idle":"2024-10-01T08:25:59.847659Z","shell.execute_reply.started":"2024-10-01T08:25:41.388857Z","shell.execute_reply":"2024-10-01T08:25:59.846309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['bg-5:55'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:25:59.850330Z","iopub.execute_input":"2024-10-01T08:25:59.850822Z","iopub.status.idle":"2024-10-01T08:25:59.864365Z","shell.execute_reply.started":"2024-10-01T08:25:59.850770Z","shell.execute_reply":"2024-10-01T08:25:59.863099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check columns for each category","metadata":{}},{"cell_type":"code","source":"#check columns for different categories\nstart_words = ['bg', 'insulin', 'carbs', 'hr', 'steps', 'cals', 'activity']  \n\n# Create a dictionary to store columns for each starting word\ncolumns_by_start = {word: [col for col in train.columns if col.startswith(word)] for word in start_words}\n\n# Display the lists of columns for each starting word\nfor word, cols in columns_by_start.items():\n    print(f\"Columns starting with '{word}': {cols}\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:25:59.865646Z","iopub.execute_input":"2024-10-01T08:25:59.866014Z","iopub.status.idle":"2024-10-01T08:25:59.885141Z","shell.execute_reply.started":"2024-10-01T08:25:59.865977Z","shell.execute_reply":"2024-10-01T08:25:59.883975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check data type - activity/time/id/p_num are categorical. the rest are float","metadata":{}},{"cell_type":"code","source":"# Create a dictionary to store columns by data type\ncolumns_by_dtype = {dtype: train.select_dtypes(include=dtype).columns.tolist() for dtype in train.dtypes.unique()}\n\n# Display the list of columns for each data type\nfor dtype, columns in columns_by_dtype.items():\n    print(f\"Data type: {dtype}, Columns: {columns}\\n\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:25:59.886369Z","iopub.execute_input":"2024-10-01T08:25:59.886757Z","iopub.status.idle":"2024-10-01T08:26:00.322107Z","shell.execute_reply.started":"2024-10-01T08:25:59.886719Z","shell.execute_reply":"2024-10-01T08:26:00.320887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"code","source":"# columns = [col for col in test.columns if col.startswith('bg')]\n# train = train[columns]\n# test = test[columns]","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:00.325069Z","iopub.execute_input":"2024-10-01T08:26:00.325474Z","iopub.status.idle":"2024-10-01T08:26:00.330113Z","shell.execute_reply.started":"2024-10-01T08:26:00.325431Z","shell.execute_reply":"2024-10-01T08:26:00.328978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:00.331739Z","iopub.execute_input":"2024-10-01T08:26:00.332158Z","iopub.status.idle":"2024-10-01T08:26:00.503406Z","shell.execute_reply.started":"2024-10-01T08:26:00.332119Z","shell.execute_reply":"2024-10-01T08:26:00.502278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train:\")\ndisplay(train.isnull().sum())\n\nprint(\"\\nTest:\")\ndisplay(test.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:00.505004Z","iopub.execute_input":"2024-10-01T08:26:00.505782Z","iopub.status.idle":"2024-10-01T08:26:01.029065Z","shell.execute_reply.started":"2024-10-01T08:26:00.505737Z","shell.execute_reply":"2024-10-01T08:26:01.027753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def impute_and_label_encode_train_test(train_df, test_df):\n   \n    for col in test_df.columns:\n        if test_df[col].dtype == 'object' and col not in [\"id\", \"p_num\", \"time\"]:  # Check if the column is of type 'object'\n            # Impute missing values with 'others' in both train and test datasets\n            train_df[col] = train_df[col].fillna('others') \n            test_df[col] = test_df[col].fillna('others')\n            \n            # Apply Label Encoding based on train data\n            le = LabelEncoder()\n            train_df[col] = le.fit_transform(train_df[col])\n            \n            # Transform test data using the same encoder\n            test_df[col] = test_df[col].apply(lambda s: s if s in le.classes_ else '<unknown>')\n      \n            new_classes = np.append(le.classes_, '<unknown>')\n            le.classes_ = new_classes\n            test_df[col] = le.transform(test_df[col])            \n   \n    return train_df, test_df\n\ndef impute_float_columns_with_median(train_df, test_df):\n    # Initialize the SimpleImputer with the median strategy\n    imputer = SimpleImputer(strategy='median')\n    \n    # Identify float columns that are not in the exclusion list\n    float_columns = [column for column in test_df.columns \n                     if test_df[column].dtype == 'float64' and column not in [\"id\", \"p_num\", \"time\"]]\n\n    # Fit the imputer on the train DataFrame and transform both train and test DataFrames\n    train_df[float_columns] = imputer.fit_transform(train_df[float_columns])\n    test_df[float_columns] = imputer.transform(test_df[float_columns])\n\n    return train_df, test_df\n\ntrain, test = impute_and_label_encode_train_test(train, test)\ntrain, test = impute_float_columns_with_median(train, test)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:01.030362Z","iopub.execute_input":"2024-10-01T08:26:01.030766Z","iopub.status.idle":"2024-10-01T08:26:19.156971Z","shell.execute_reply.started":"2024-10-01T08:26:01.030730Z","shell.execute_reply":"2024-10-01T08:26:19.155835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:19.158110Z","iopub.execute_input":"2024-10-01T08:26:19.158457Z","iopub.status.idle":"2024-10-01T08:26:19.338679Z","shell.execute_reply.started":"2024-10-01T08:26:19.158413Z","shell.execute_reply":"2024-10-01T08:26:19.337488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:19.340106Z","iopub.execute_input":"2024-10-01T08:26:19.340455Z","iopub.status.idle":"2024-10-01T08:26:19.374790Z","shell.execute_reply.started":"2024-10-01T08:26:19.340419Z","shell.execute_reply":"2024-10-01T08:26:19.373566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#time feature engineering\ndef time_fe(df):\n\n    df['time'] = pd.to_datetime(df['time'], format='%H:%M:%S').dt.time\n    new_columns = pd.DataFrame({\n    'hour': df['time'].apply(lambda x: x.hour),\n    'minute': df['time'].apply(lambda x: x.minute),\n    'second': df['time'].apply(lambda x: x.second)\n   \n    })\n\n    # Concatenate the new columns with the original DataFrame\n    df = pd.concat([df, new_columns], axis=1)\n\n    df = df.drop([\"time\"],axis=1)\n    \n    return df\n\ndef categorize_time(hour):\n    if 5 <= hour < 12:\n        return 0\n    elif 12 <= hour < 17:\n        return 1\n    elif 17 <= hour < 21:\n        return 2\n    else:\n        return 3\n\ntrain = time_fe(train)\ntest = time_fe(test)\n\ntrain['time_of_day'] = train['hour'].apply(categorize_time)\ntest['time_of_day'] = test['hour'].apply(categorize_time)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:19.376268Z","iopub.execute_input":"2024-10-01T08:26:19.376631Z","iopub.status.idle":"2024-10-01T08:26:21.061460Z","shell.execute_reply.started":"2024-10-01T08:26:19.376595Z","shell.execute_reply":"2024-10-01T08:26:21.060354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #rolling window feature engineering for 'bg', 'insulin', 'carbs', 'hr', 'steps', 'cals'\n# import warnings\n\n# # Suppress only FutureWarnings\n# warnings.simplefilter(action='ignore', category=FutureWarning)\n\n# def create_rolling_features_across_columns(df, prefixes, window_sizes):\n#     # Create a dictionary to hold the new rolling feature columns\n#     rolling_features = {}\n\n#     # Loop through each prefix (e.g., 'bg')\n#     for prefix in prefixes:\n#         # Extract columns starting with the current prefix        \n#         columns = [col for col in df.columns if col.startswith(prefix) and not (col == 'bg+1:00')]\n        \n#         # Check if there are any columns for the current prefix\n#         if columns:\n#             # Loop through each window size\n#             for window_size in window_sizes:\n#                 # Calculate rolling mean across columns\n#                 rolling_mean = df[columns].rolling(window=window_size, axis=1).mean()\n                \n#                 # Add the rolling mean feature to the new columns\n#                 for i, col in enumerate(columns[window_size-1:], start=window_size-1):\n#                     rolling_features[f'{prefix}_rolling_mean_{window_size}_{col}'] = rolling_mean.iloc[:, i]\n\n#     # Convert the dictionary to a DataFrame and concatenate it with the original DataFrame\n#     rolling_df = pd.DataFrame(rolling_features, index=df.index)\n#     df = pd.concat([df, rolling_df], axis=1)\n    \n#     return df\n\n# # Example usage for 'bg' prefix and rolling window sizes 3 and 2\n# prefixes = ['bg', 'insulin', 'carbs', 'hr', 'steps', 'cals'] #, \n# window_sizes = [3,6,9,12] # \n\n# # Assuming df is your DataFrame\n# train = create_rolling_features_across_columns(train, prefixes, window_sizes)\n# test = create_rolling_features_across_columns(test, prefixes, window_sizes)\n# train","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:21.062893Z","iopub.execute_input":"2024-10-01T08:26:21.063365Z","iopub.status.idle":"2024-10-01T08:26:21.070476Z","shell.execute_reply.started":"2024-10-01T08:26:21.063316Z","shell.execute_reply":"2024-10-01T08:26:21.069345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_glucose_levels(df, lower_bound, upper_bound):\n    \"\"\"\n    Encodes glucose levels into 0 or 1 based on specified range.\n    \n    Parameters:\n    df (pd.DataFrame): The input DataFrame.\n    prefix (str): The prefix to filter columns (e.g., 'bg').\n    lower_bound (float): The lower bound for encoding 0.\n    upper_bound (float): The upper bound for encoding 0.\n    \n    Returns:\n    pd.DataFrame: The DataFrame with encoded columns.\n    \"\"\"\n    # Filter columns that start with the prefix (e.g., 'bg')\n    glucose_columns = [col for col in df.columns if col.startswith('bg') and not (col == 'bg+1:00')]\n    \n    for col in glucose_columns:\n        # Encode 0 if value is between lower_bound and upper_bound, else 1\n        df[col] = df[col].apply(lambda x: 0 if lower_bound <= x <= upper_bound else 1)\n    \n    return df\n\n# Apply encoding for train and test datasets\ntrain = encode_glucose_levels(train, lower_bound=0.0, upper_bound=9.0)\ntest = encode_glucose_levels(test,lower_bound=0.0, upper_bound=9.0)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:21.072035Z","iopub.execute_input":"2024-10-01T08:26:21.072423Z","iopub.status.idle":"2024-10-01T08:26:27.204129Z","shell.execute_reply.started":"2024-10-01T08:26:21.072387Z","shell.execute_reply":"2024-10-01T08:26:27.203110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.207963Z","iopub.execute_input":"2024-10-01T08:26:27.208876Z","iopub.status.idle":"2024-10-01T08:26:27.351940Z","shell.execute_reply.started":"2024-10-01T08:26:27.208832Z","shell.execute_reply":"2024-10-01T08:26:27.350874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().any().any()","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.353312Z","iopub.execute_input":"2024-10-01T08:26:27.353638Z","iopub.status.idle":"2024-10-01T08:26:27.443567Z","shell.execute_reply.started":"2024-10-01T08:26:27.353605Z","shell.execute_reply":"2024-10-01T08:26:27.442442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isna().any().any()","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.445029Z","iopub.execute_input":"2024-10-01T08:26:27.445412Z","iopub.status.idle":"2024-10-01T08:26:27.458183Z","shell.execute_reply.started":"2024-10-01T08:26:27.445372Z","shell.execute_reply":"2024-10-01T08:26:27.456994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.459938Z","iopub.execute_input":"2024-10-01T08:26:27.460388Z","iopub.status.idle":"2024-10-01T08:26:27.551996Z","shell.execute_reply.started":"2024-10-01T08:26:27.460331Z","shell.execute_reply":"2024-10-01T08:26:27.550841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.553555Z","iopub.execute_input":"2024-10-01T08:26:27.553888Z","iopub.status.idle":"2024-10-01T08:26:27.579454Z","shell.execute_reply.started":"2024-10-01T08:26:27.553853Z","shell.execute_reply":"2024-10-01T08:26:27.578301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[\"bg+1:00\"]\nX  = train.drop([\"p_num\",\"id\", \"bg+1:00\"],axis=1)\n\n# X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\nX_test = test.drop([\"p_num\",\"id\"],axis=1)\n\n# print(X_train.shape)\n# print(y_train.shape)\n\n# Initialize the MinMaxScaler\n# scaler = MinMaxScaler(feature_range=(0, 1))\n\n# Fit the scaler on the training data only\n# X_train = pd.DataFrame(scaler.fit_transform(X_train),columns=X.columns)\n\n# # Apply the scaler to validation and test data (without fitting again)\n# X_val= pd.DataFrame(scaler.transform(X_val),columns=X.columns)\n# X_test = pd.DataFrame(scaler.transform(X_test),columns=X.columns)\n# X = pd.DataFrame(scaler.fit_transform(X),columns=train.drop([\"p_num\",\"id\", \"bg+1:00\"],axis=1).columns)\n# X_test = pd.DataFrame(scaler.transform(X_test),columns=train.drop([\"p_num\",\"id\", \"bg+1:00\"],axis=1).columns)\n\n# # Save the scaler to a file\n# scaler_filename = \"minmax_scaler.pkl\"\n# joblib.dump(scaler, scaler_filename)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.580992Z","iopub.execute_input":"2024-10-01T08:26:27.581953Z","iopub.status.idle":"2024-10-01T08:26:27.810544Z","shell.execute_reply.started":"2024-10-01T08:26:27.581878Z","shell.execute_reply":"2024-10-01T08:26:27.809365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.811875Z","iopub.execute_input":"2024-10-01T08:26:27.812244Z","iopub.status.idle":"2024-10-01T08:26:27.889155Z","shell.execute_reply.started":"2024-10-01T08:26:27.812207Z","shell.execute_reply":"2024-10-01T08:26:27.887996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.890503Z","iopub.execute_input":"2024-10-01T08:26:27.890839Z","iopub.status.idle":"2024-10-01T08:26:27.915621Z","shell.execute_reply.started":"2024-10-01T08:26:27.890806Z","shell.execute_reply":"2024-10-01T08:26:27.914580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_highly_correlated_features(df, threshold):\n    # Calculate the correlation matrix\n    corr_matrix = df.corr().abs()\n    \n    # Create a mask for the upper triangle\n    upper_triangle = corr_matrix.where(~np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n    \n    # Find correlated pairs excluding correlations of a feature with itself\n    correlated_pairs = [(row, column) \n                        for column in upper_triangle.columns for row in upper_triangle.index \n                        if upper_triangle.loc[row, column] > threshold and row != column]\n    \n    correlated_pairs_df = pd.DataFrame(correlated_pairs, columns=['Feature 1', 'Feature 2'])\n    \n    display(correlated_pairs_df)\n    \n    # Set to store columns to drop\n    columns_to_drop = set()\n\n    # Iterate over the correlated pairs and keep only one feature from each pair\n    for row, column in correlated_pairs:\n        if column not in columns_to_drop and not column.endswith('00'):\n            columns_to_drop.add(column)  # Keep 'row' and drop 'column'\n    \n    # Drop the highly correlated features from the DataFrame\n    df = df.drop(columns=list(columns_to_drop))\n    \n    print(f\"Removed columns: {list(columns_to_drop)}\")\n    \n    return df\n\nX = remove_highly_correlated_features(X, threshold=0.95)\nX","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:26:27.916950Z","iopub.execute_input":"2024-10-01T08:26:27.917340Z","iopub.status.idle":"2024-10-01T08:28:34.425627Z","shell.execute_reply.started":"2024-10-01T08:26:27.917304Z","shell.execute_reply":"2024-10-01T08:28:34.424445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Selection","metadata":{}},{"cell_type":"code","source":"# #ANOVA\n# from sklearn.feature_selection import SelectKBest, f_regression\n# import pandas as pd\n\n# # Apply SelectKBest with ANOVA F-test for regression\n# selector = SelectKBest(score_func=f_regression, k='all')  # Get scores for all features\n# selector.fit(X, y)\n\n# # Get F-scores and corresponding feature names\n# f_scores = selector.scores_\n# feature_names = X.columns\n\n# # Create a DataFrame for easy sorting\n# importance_df = pd.DataFrame({'Feature': feature_names, 'F-score': f_scores})\n\n# # Sort the DataFrame by F-score in decreasing order\n# importance_df = importance_df.sort_values(by='F-score', ascending=False)\n\n# display(importance_df)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:28:34.426825Z","iopub.execute_input":"2024-10-01T08:28:34.427167Z","iopub.status.idle":"2024-10-01T08:28:35.048182Z","shell.execute_reply.started":"2024-10-01T08:28:34.427133Z","shell.execute_reply":"2024-10-01T08:28:35.046978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from boruta import BorutaPy\nfrom xgboost import XGBRegressor\n\nxgb = XGBRegressor(seed=42)\nboruta_selector = BorutaPy(estimator=xgb, n_estimators='auto', random_state=42, verbose=0,two_step=True)\n\n# Fit Boruta\nboruta_selector.fit(X.values, y.values)\n\n# Get the selected features\nselected_features = X.columns[boruta_selector.support_].to_list()\nnon_selected_features = X.columns[~boruta_selector.support_].tolist()\nprint(\"Selected features:\\n\", selected_features)\nprint(\"Length of selected features:\\n\", len(selected_features))\nprint(\"\\n Non Selected features:\\n\", non_selected_features)\nprint(\"Length of non selected features:\\n\", len(non_selected_features))\n\n# Transform the dataset\nX_selected = boruta_selector.transform(X.values)\n\n# Train the model on the selected features\nxgb.fit(X_selected, y)\n\nfeature_importances = xgb.feature_importances_\n\nfeature_importance_df = pd.DataFrame({'Feature': selected_features, 'Importance': feature_importances})\nfeature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\ndisplay(feature_importance_df)\n\nselected_features = list(feature_importance_df['Feature'])\n# selected_features = list(importance_df['Feature'].head(10))\n\nX = X[selected_features]\nX_test = X_test[selected_features]","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:28:35.050151Z","iopub.execute_input":"2024-10-01T08:28:35.050765Z","iopub.status.idle":"2024-10-01T08:36:33.399308Z","shell.execute_reply.started":"2024-10-01T08:28:35.050710Z","shell.execute_reply":"2024-10-01T08:36:33.397500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model building","metadata":{}},{"cell_type":"code","source":"import optuna\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import mean_squared_error\nimport numpy as np\n\ndef objective(trial):\n    # Define hyperparameter search space\n    params = {\n        'objective': 'reg:squarederror',\n        'n_estimators': trial.suggest_int('n_estimators', 300, 1000, step=100),\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'learning_rate': trial.suggest_float('learning_rate', 1e-2, 0.1, log=True),\n        'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n        'gamma': trial.suggest_float('gamma', 0, 2), \n        'reg_alpha': trial.suggest_float('reg_alpha', 0, 5),  \n        'reg_lambda': trial.suggest_float('reg_lambda', 0, 5),  \n        'verbosity': 0,\n        'tree_method': 'gpu_hist',  \n        'predictor': 'gpu_predictor' \n    }\n\n    # Create the model\n    model = XGBRegressor(**params)\n\n    # Perform cross-validation and calculate RMSE (negative because we want to minimize)\n    rmse = -np.mean(cross_val_score(model, X, y,\n                                     scoring='neg_root_mean_squared_error',\n                                     cv=2, n_jobs=-1))\n\n    return rmse \n\nstart_time = time.time()  \n# Create a study object and optimize\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n\n# Get the best parameters and score\nbest_params = study.best_params\nbest_rmse = study.best_value\n\nprint(\"Best hyperparameters:\", best_params)\nprint(\"Best RMSE:\", best_rmse)\n\n# Train the model using the best parameters\n\nbest_model = XGBRegressor(**best_params, verbosity=0)\nbest_model.fit(X, y)\nend_time = time.time()  \n\n\nprint(f\"Time taken: {(end_time - start_time) / 60:.2f} minutes\")","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:36:33.400638Z","iopub.status.idle":"2024-10-01T08:36:33.401246Z","shell.execute_reply.started":"2024-10-01T08:36:33.400947Z","shell.execute_reply":"2024-10-01T08:36:33.400980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([test[\"id\"],pd.DataFrame(best_model.predict(X_test),columns=['bg+1:00'])],axis=1)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:36:33.403126Z","iopub.status.idle":"2024-10-01T08:36:33.403699Z","shell.execute_reply.started":"2024-10-01T08:36:33.403406Z","shell.execute_reply":"2024-10-01T08:36:33.403434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T08:36:33.405263Z","iopub.status.idle":"2024-10-01T08:36:33.405668Z","shell.execute_reply.started":"2024-10-01T08:36:33.405475Z","shell.execute_reply":"2024-10-01T08:36:33.405496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}