\n\ncarbs and activity have a great deal of empty data, but the distribution of the data we have is on the whole day.","metadata":{}},{"cell_type":"code","source":"# Aggregate data by patient and plot the distribution\ndf_patient_avg = raw_data.groupby('p_num')['target'].mean().reset_index()\n\nplt.figure(figsize=(12, 5))\nsns.violinplot(data=df_patient_avg, x='p_num', y='target')\nplt.title('Average bg+1:00 by Patient')\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:15:29.041860Z","iopub.execute_input":"2025-01-05T17:15:29.042341Z","iopub.status.idle":"2025-01-05T17:15:29.597994Z","shell.execute_reply.started":"2025-01-05T17:15:29.042295Z","shell.execute_reply":"2025-01-05T17:15:29.596837Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##
\n\nFeature importance analysis has shown that the prediction is influences only by data from the past 45 mins / 1 hour.
\nWe can seize this opportunity to increase our dataset size by 6 times by changing the problem statement.","metadata":{}},{"cell_type":"code","source":"time_series = {\n '0': ['_0_55', '_0_50', '_0_45', '_0_40', '_0_35', '_0_30', '_0_25', '_0_20', '_0_15', '_0_10', '_0_05', '_0_00'],\n '1': ['_1_55', '_1_50', '_1_45', '_1_40', '_1_35', '_1_30', '_1_25', '_1_20', '_1_15', '_1_10', '_1_05', '_1_00'],\n '2': ['_2_55', '_2_50', '_2_45', '_2_40', '_2_35', '_2_30', '_2_25', '_2_20', '_2_15', '_2_10', '_2_05', '_2_00'],\n '3': ['_3_55', '_3_50', '_3_45', '_3_40', '_3_35', '_3_30', '_3_25', '_3_20', '_3_15', '_3_10', '_3_05', '_3_00'],\n '4': ['_4_55', '_4_50', '_4_45', '_4_40', '_4_35', '_4_30', '_4_25', '_4_20', '_4_15', '_4_10', '_4_05', '_4_00'],\n '5': ['_5_55', '_5_50', '_5_45', '_5_40', '_5_35', '_5_30', '_5_25', '_5_20', '_5_15', '_5_10', '_5_05', '_5_00']\n}\n\ndef extract_sub_row(row, time_serie_suffixes, target_column, time_delta):\n \n # transform the row into a dictionary\n row_dict = row.to_dict()\n \n # create subrow object\n sub_row = {}\n \n # add patient number\n sub_row['p_num'] = row['p_num']\n \n # set the real time, by subtracting the delta from the time column\n sub_row['time'] = pd.to_datetime(row['time'], format='%H:%M:%S') - pd.Timedelta(hours=time_delta)\n\n # set the new target\n sub_row['target'] = row[target_column]\n \n # add columns containing the provided time series suffix list\n for key, value in row_dict.items():\n \n # check if the column contains one of the time series suffixes\n for index, suffix in enumerate(time_serie_suffixes):\n \n if suffix in key:\n # get metric name from key(bg_X_XX)\n metric_name = key.split('_')[0]\n \n # generate new column name\n new_column_name = f'{metric_name}_{str(index)}'\n \n # add column to the row\n sub_row[new_column_name] = value\n \n return sub_row\n \ndef split_data(df):\n \"\"\"\n Split a row with 6 hours of data into 6 rows with 1 hour of data\n :param df:\n :return:\n \"\"\" \n rows = []\n \n # for each row in the dataset\n for idx, row in df.iterrows():\n # extract 6 one hour rows\n sub_row_0 = extract_sub_row(row, time_series['0'], 'target', 0)\n rows.append(sub_row_0)\n for i in range(1,6):\n target_column = f'bg_{str(i - 1)}_00'\n sub_row = extract_sub_row(row, time_series[str(i)], target_column, i)\n rows.append(sub_row)\n \n return pd.DataFrame(rows).sort_values(by=['p_num', 'time']).reset_index(drop=True)\n\ndef drop_prediction_columns(df):\n \"\"\"\n Drops all the columns with data older than one hour\n :param df:\n :return:\n \"\"\"\n cols_to_drop = []\n for column in df.columns:\n # rename first hour data columns\n for idx, suffix in enumerate(time_series['0']):\n if suffix in column:\n # get metric name from key(bg_X_XX)\n metric_name = column.split('_')[0]\n # generate new column name\n new_column_name = f'{metric_name}_{str(idx)}'\n # add new column\n df[new_column_name] = df[column]\n # remove old column\n cols_to_drop.append(column)\n continue\n \n for i in range(1,6):\n if any(time_serie_suffix in column for time_serie_suffix in time_series[str(i)]):\n cols_to_drop.append(column)\n df.drop(cols_to_drop,axis=1,inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:15:29.599698Z","iopub.execute_input":"2025-01-05T17:15:29.600061Z","iopub.status.idle":"2025-01-05T17:15:29.616509Z","shell.execute_reply.started":"2025-01-05T17:15:29.600028Z","shell.execute_reply":"2025-01-05T17:15:29.615291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mesas_cols=['bg', 'insulin','steps','hr','cals']\ndivisible_cols=['bg','hr']\n\ndef create_features(df):\n # Generate mean, std and zscore of latest value\n for col in mesas_cols:\n # calculate mean\n df[f'mean_patient_{col}_11'] = df.groupby(['p_num'])[f\"{col}_11\"].transform('mean')\n # calculate standard deviation\n df[f'std_patient_{col}_11'] = df.groupby(['p_num'])[f\"{col}_11\"].transform('std')\n # z-score normalization\n df[f'norm_patient_{col}_11'] = (df[f'{col}_11'] - df[f'mean_patient_{col}_11']) / df[f'std_patient_{col}_11']\n \n\n # Pairwise Feature Interactions on 3 latest values\n for i in range(len(mesas_cols)):\n for j in range(i + 1, len(mesas_cols)):\n for k in [1,2,3]:\n time_index = 12 - k\n # bg_11_plus_hr_11 = bg_11 + hr_11\n df[f\"{mesas_cols[i]}_{time_index}_plus_{mesas_cols[j]}_{time_index}\"] = df[f\"{mesas_cols[i]}_{time_index}\"] + df[f\"{mesas_cols[j]}_{time_index}\"]\n # bg_11_minus_hr_11 = bg_11 - hr_11\n df[f\"{mesas_cols[i]}_{time_index}_minus_{mesas_cols[j]}_{time_index}\"] = df[f\"{mesas_cols[i]}_{time_index}\"] - df[f\"{mesas_cols[j]}_{time_index}\"]\n # bg_11_multiplied_hr_11 = bg_11 * hr_11\n df[f\"{mesas_cols[i]}_{time_index}_multiplied_{mesas_cols[j]}_{time_index}\"] = df[f\"{mesas_cols[i]}_{time_index}\"] * df[f\"{mesas_cols[j]}_{time_index}\"]\n # bg_11_divided_hr_11 = bg_11 / hr_11 + 1e-15\n df[f\"{mesas_cols[i]}_{time_index}_divided_{mesas_cols[j]}_{time_index}\"] = df[f\"{mesas_cols[i]}_{time_index}\"] / (df[f\"{mesas_cols[j]}_{time_index}\"] + 1e-15)\n \n # bg_11_minus_hr_11_minus_bg_10_minus_hr_10 = bg_11_minus_hr_11 - bg_10_minus_hr_10\n df[f\"{mesas_cols[i]}_11_minus_{mesas_cols[j]}_11_minus_{mesas_cols[i]}_10_minus_{mesas_cols[j]}_10\"]=\\\n df[f\"{mesas_cols[i]}_11_minus_{mesas_cols[j]}_11\"] - df[f\"{mesas_cols[i]}_10_minus_{mesas_cols[j]}_10\"]\n \n # compute differences and combinations of all values\n for col in mesas_cols:\n for i in range(1,12):\n # bg_11_minus_bg_10 = bg_11 - bg_10\n df[f\"{col}_{i}_minus_{col}_{i-1}\"]=df[f\"{col}_{i}\"] - df[f\"{col}_{i-1}\"]\n \n # bg_11_plus_bg_10 = bg_11 + bg_10\n df[f'{col}_{i}_plus_{col}_{i-1}'] = df[f'{col}_{i}'] + df[f'{col}_{i-1}']\n \n # bg_11_multiplied_bg_10 = bg_11 * bg_10\n df[f'{col}_{i}_multiplied_{col}_{i-1}'] = df[f'{col}_{i}'] * df[f'{col}_{i-1}']\n \n # compute division only on worthy columns\n for col in divisible_cols:\n for i in range(1,12): \n # bg_11_divided_bg_10 = bg_11 / bg_10\n df[f'{col}_{i}_divided_{col}_{i-1}'] = df[f'{col}_{i}'] / df[f'{col}_{i-1}']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:15:29.617822Z","iopub.execute_input":"2025-01-05T17:15:29.618137Z","iopub.status.idle":"2025-01-05T17:15:29.634774Z","shell.execute_reply.started":"2025-01-05T17:15:29.618109Z","shell.execute_reply":"2025-01-05T17:15:29.633486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"activity_dictionary = {'None': 0, 'Yoga': 1, 'Walking': 2, 'Walk': 2, 'Dancing': 3, 'Zumba': 4,\n 'Strength training': 5,\n 'Weights': 6, 'Aerobic Workout': 7, 'Workout': 8, 'HIIT': 9, 'Run': 10, 'Running': 10,\n 'Bike': 11, 'Outdoor Bike': 11, 'Stairclimber': 12, 'Spinning': 13, 'Swim': 14,\n 'Swimming': 14,\n 'Tennis': 15, 'Indoor climbing': 16, 'Hike': 17, 'Sport': 18}\n\n# Create a dictionary that maps each patient to a number\npartecipants_dictionary = {'p01': 1, 'p02': 2, 'p03': 3, 'p04': 4, 'p05': 5, 'p06': 6, 'p07': 7, 'p08': 8, 'p09': 9,\n 'p10': 10, 'p11': 11, 'p12': 12, 'p13': 13, 'p14': 14, 'p15': 15, 'p16': 16, 'p17': 17,\n 'p18': 18, 'p19': 19, 'p20': 20, 'p21': 21, 'p22': 22, 'p23': 23, 'p24': 24}\n\ndef drop_features(X):\n # drop carbs\n X.drop([c for c in X.columns if 'carbs' in c],axis=1,inplace=True)\n\n\ndef transform_time(X):\n # extract a hour and minute series\n if not is_datetime(X['time']):\n X['time'] = pd.to_datetime(X['time'], format='%H:%M:%S')\n X[\"hour\"] = X['time'].dt.hour\n X[\"minute\"] = X['time'].dt.minute\n\n # create half day sin cos wave\n X['sin_halfday'] = np.sin(2 * np.pi * X['hour'] / 12)\n X['cos_halfday'] = np.cos(2 * np.pi * X['hour'] / 12)\n # create a hour sin and cos wave\n X['hour_sin'] = np.sin(2 * np.pi * X[\"hour\"] / 24)\n X['hour_cos'] = np.cos(2 * np.pi * X[\"hour\"] / 24)\n # create a minute sin and cos wave\n X['minute_sin'] = np.sin(2 * np.pi * X[\"minute\"] / 60)\n X['minute_cos'] = np.cos(2 * np.pi * X[\"minute\"] / 60)\n\n # remove time\n X.drop(['time'], axis=1, inplace=True)\n\n\ndef encode_and_0fill(X):\n steps_columns = []\n activity_columns = []\n carbs_columns = []\n cals_columns = []\n\n for column in X.columns:\n # if the column is a metric with a survey time\n if 'steps_' in column:\n steps_columns.append(column)\n if 'activity_' in column:\n activity_columns.append(column)\n if 'carbs_' in column:\n carbs_columns.append(column)\n if 'cals_' in column:\n cals_columns.append(column)\n\n X[activity_columns] = X[activity_columns].fillna('None') # assume no activity when empty\n\n # apply static encoding for activities\n for col in activity_columns:\n # Assign values based on the dictionary and set -1 for empty values\n X[col] = X[col].apply(lambda x: activity_dictionary.get(x, -1) if x != '' else -1)\n\n # apply static encoding for patients\n # Assign values based on the dictionary and set -1 for empty values\n X['p_num'] = X['p_num'].apply(lambda x: partecipants_dictionary.get(x, -1) if x != '' else -1)\n\n X[steps_columns] = X[steps_columns].fillna(0) # assume 0 steps when empty\n X[carbs_columns] = X[carbs_columns].fillna(0) # assume 0 carbohydrate intake when empty\n X[cals_columns] = X[cals_columns].fillna(0) # assume 0 calories burned when empty\n\n\ndef fill_time_series(X):\n # create a dictionary of metrics and their corresponding columns\n metric_columns = {}\n for column in X.columns:\n if '_' in column:\n # get the metric name\n metric_name = column.split(\"_\")[0]\n # get the array of existing columns with the same metrics (or create it) and add the current column\n metric_columns_names = metric_columns.get(metric_name) or []\n metric_columns_names.append(column)\n # save the new array on the dictionary\n metric_columns[metric_name] = metric_columns_names\n\n # iterate metrics\n for key, value in metric_columns.items():\n # Sort the metric columns in descending order\n columns = sorted(value, key=lambda x: '_'.join(x.split('_')[1:]), reverse=True)\n\n # Fill missing values using backfill (and forward fill if the latest values are empty)\n X[columns] = X[columns].bfill()\n X[columns] = X[columns].ffill()\n\ndef delete_strong_correlated_columns(X):\n # reduce overfitting, strong correlations between nearest timestep features\n X.drop([f'bg_{i}'for i in range(12) if i % 2 != 0], axis=1, inplace=True)\n\ndef preprocess_data(X):\n #drop_features(X)\n encode_and_0fill(X)\n fill_time_series(X)\n drop_prediction_columns(X)\n #create_features(X)\n transform_time(X)\n #delete_strong_correlated_columns(X)\n\ndef preprocess_train_data(X):\n #drop_features(X)\n encode_and_0fill(X)\n fill_time_series(X)\n X = split_data(X)\n #create_features(X)\n transform_time(X)\n #delete_strong_correlated_columns(X)\n return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:15:29.636687Z","iopub.execute_input":"2025-01-05T17:15:29.637105Z","iopub.status.idle":"2025-01-05T17:15:29.660198Z","shell.execute_reply.started":"2025-01-05T17:15:29.637070Z","shell.execute_reply":"2025-01-05T17:15:29.658859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X['target'] = y\nX = preprocess_train_data(X)\nX.to_csv('preprocessed_train.csv')\ny = X['target']\nX = X.drop(['target'], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:15:29.661704Z","iopub.execute_input":"2025-01-05T17:15:29.662079Z","iopub.status.idle":"2025-01-05T17:38:49.102556Z","shell.execute_reply.started":"2025-01-05T17:15:29.662044Z","shell.execute_reply":"2025-01-05T17:38:49.100020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('total columns: ', len(X.columns.values))\n\nfeatures_with_missing_values = [col for col in X.columns if X[col].isnull().any()]\n\nprint('number of columns with missing values: ', len(features_with_missing_values))\nprint(features_with_missing_values)\nprint('--- Percentage of Missing Values ---')\nprint(100*X[features_with_missing_values].isnull().sum() / X.shape[0])\n\ncategorical_cols = [cname for cname in X.columns if X[cname].dtype == \"object\"]\nprint('number of categorical columns: ', len(categorical_cols))\nprint(categorical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:38:49.105199Z","iopub.execute_input":"2025-01-05T17:38:49.105636Z","iopub.status.idle":"2025-01-05T17:38:49.206838Z","shell.execute_reply.started":"2025-01-05T17:38:49.105598Z","shell.execute_reply":"2025-01-05T17:38:49.205286Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"
\n\nDataframe successfully cleaned up, no need for further pipelines.","metadata":{}},{"cell_type":"markdown","source":"##
Train model
","metadata":{}},{"cell_type":"code","source":"# instantiate data pipeline\nfrom src.pipelines.empty_dt_pipeline import EmptyDTPipeline\npipeline = EmptyDTPipeline(X)\n\n# pick a model, and a trainer\nmodel_type = XGBRegressorWrapper()\ntrainer = AccurateCrossTrainer(pipeline, model_type, metric=AccuracyMetric.RMSE)\n\n# optimizing parameters worsens performance for this dataset\noptimized_params = {\n 'objective': 'reg:squarederror',\n 'learning_rate': 0.03,\n 'n_jobs': -1,\n}\n\n# Train model\n_, boost_rounds, _ = trainer.validate_model(X, y, log_level=0, params=optimized_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:38:49.208141Z","iopub.execute_input":"2025-01-05T17:38:49.208479Z","iopub.status.idle":"2025-01-05T18:21:01.694041Z","shell.execute_reply.started":"2025-01-05T17:38:49.208436Z","shell.execute_reply":"2025-01-05T18:21:01.691840Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer.show_feature_importance(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:21:01.696860Z","iopub.execute_input":"2025-01-05T18:21:01.698215Z","iopub.status.idle":"2025-01-05T18:21:02.285848Z","shell.execute_reply.started":"2025-01-05T18:21:01.698169Z","shell.execute_reply":"2025-01-05T18:21:02.284645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer.show_loss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:21:02.287198Z","iopub.execute_input":"2025-01-05T18:21:02.287562Z","iopub.status.idle":"2025-01-05T18:21:02.598124Z","shell.execute_reply.started":"2025-01-05T18:21:02.287530Z","shell.execute_reply":"2025-01-05T18:21:02.596824Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##
Train complete model on all data
","metadata":{}},{"cell_type":"code","source":"# fit complete_model on all data from the training data\ncomplete_model = trainer.train_model(X, y, iterations=boost_rounds, params=optimized_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:21:02.599776Z","iopub.execute_input":"2025-01-05T18:21:02.600126Z","iopub.status.idle":"2025-01-05T18:24:23.747192Z","shell.execute_reply.started":"2025-01-05T18:21:02.600093Z","shell.execute_reply":"2025-01-05T18:24:23.745897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pickle components, to experiment later\nimport shutil\nfrom src.pipelines.dt_pipeline import save_data_model\nfrom src.trainers.trainer import save_model\n\n# save model file for current dataset on target directory\nprint(\"Saving data model...\")\nsave_data_model(X)\n\n# save trained pipeline on target directory\nprint(\"Saving pipeline...\")\npipeline.save_pipeline()\n\n# save model on target directory\nprint(\"Saving fitted model...\")\nsave_model(complete_model)\n\nshutil.copytree('../target', 'target')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T19:04:51.316167Z","iopub.execute_input":"2025-01-05T19:04:51.316648Z","iopub.status.idle":"2025-01-05T19:04:51.412707Z","shell.execute_reply.started":"2025-01-05T19:04:51.316610Z","shell.execute_reply":"2025-01-05T19:04:51.411330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##
Make predictions on the competition data, and generate submission
","metadata":{}},{"cell_type":"code","source":"# Load the data\ntest_data_path = '/kaggle/input/brist1d/test.csv'\ntest_data = pd.read_csv(test_data_path, index_col='id')\ntest_data = test_data.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '_', x))\npreprocess_data(test_data)\n\n# make predictions which we will submit. \ntest_preds = complete_model.predict(test_data)\n\n# Save predictions in the format used for competition scoring\noutput = pd.DataFrame({'id': test_data.index,\n 'bg+1:00': test_preds})\noutput.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:24:23.813442Z","iopub.execute_input":"2025-01-05T18:24:23.813849Z","iopub.status.idle":"2025-01-05T18:24:24.719590Z","shell.execute_reply.started":"2025-01-05T18:24:23.813814Z","shell.execute_reply":"2025-01-05T18:24:24.718456Z"}},"outputs":[],"execution_count":null}]}