{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":" #Import necessary libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:08.296827Z","iopub.execute_input":"2024-11-29T21:15:08.297252Z","iopub.status.idle":"2024-11-29T21:15:09.377750Z","shell.execute_reply.started":"2024-11-29T21:15:08.297214Z","shell.execute_reply":"2024-11-29T21:15:09.376271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Loading dataset\ntrain = pd.read_csv(\"/kaggle/input/brist1d/train.csv\", low_memory=False)\ntest = pd.read_csv(\"/kaggle/input/brist1d/test.csv\", low_memory=False)\nsample = pd.read_csv(\"/kaggle/input/brist1d/sample_submission.csv\", low_memory=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:09.380103Z","iopub.execute_input":"2024-11-29T21:15:09.380943Z","iopub.status.idle":"2024-11-29T21:15:30.669508Z","shell.execute_reply.started":"2024-11-29T21:15:09.380846Z","shell.execute_reply":"2024-11-29T21:15:30.668281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Preprocessing Summary\n\n### **1. Mapping `p_num` to Numeric Values**  \nThe `p_num` column, which contains categorical identifiers, is mapped to unique integer values. This makes the column suitable for machine learning models that require numerical input.\n\n### **2. Dropping the id Column**\nThe id column, which serves as a unique identifier, is removed if it exists, as it does not provide predictive value for modeling.\n\n### Purpose:\n- Mapping p_num: Converts categorical data into a numerical format.\n- Dropping id: Removes redundant columns that don't contribute to predictions.","metadata":{}},{"cell_type":"code","source":"# Assuming train is a DataFrame\nmapping_dict = {p: idx for idx, p in enumerate(train['p_num'].unique())}\n\n# Map p_num and process time directly\ntrain['p_num'] = train['p_num'].map(mapping_dict)\n# Drop the 'id' column if it exists\nif 'id' in train.columns:\n    train.drop(columns=['id'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:30.670993Z","iopub.execute_input":"2024-11-29T21:15:30.671456Z","iopub.status.idle":"2024-11-29T21:15:31.071618Z","shell.execute_reply.started":"2024-11-29T21:15:30.671407Z","shell.execute_reply":"2024-11-29T21:15:31.070362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Assuming train is a DataFrame\nmapping_dict = {p: idx for idx, p in enumerate(test['p_num'].unique())}\n\n# Map p_num and process time directly\ntest['p_num'] = test['p_num'].map(mapping_dict)\n\n\nif 'id' in test.columns:\n    test.drop(columns=['id'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:31.073873Z","iopub.execute_input":"2024-11-29T21:15:31.074274Z","iopub.status.idle":"2024-11-29T21:15:31.093781Z","shell.execute_reply.started":"2024-11-29T21:15:31.074237Z","shell.execute_reply":"2024-11-29T21:15:31.092414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Preprocessing \r\n\r\n### **1. Converting `time` to Datetime Format**  \r\nThe `time` column, which contains date and time information, is converted to the `datetime` format. This ensures that we can perform time-based operations such as extracting hours or calculating cyclical features. Invalid entries in the `time` column are converted to `NaT` (Not a Time) to prevent errors.\r\n\r\n### **2. Creating Time-Based Features: `sin_hour` and `cos_hour`**  \r\nTo capture the cyclical nature of the hours in a day, two new features are created: `sin_hour` and `cos_hour`. These features are generated by applying sine and cosine transformations to the hour of the day. This encoding preserves the cyclical nature of time (i.e., 23:00 is close to 00:00).\r\n\r\n### **3. Dropping the `time` Column**  \r\nAfter generating the `sin_hour` and `cos_hour` features, the original `time` column is dropped, as the new features now encode the necessary time-related information for modeling.\r\n\r\n### Purpose:\r\n- Converting `time` to `datetime`: Allows for easy manipulation and extraction of time-based information.\r\n- Creating `sin_hour` and `cos_hour`: Encodes the cyclical nature of time for better model performance.\r\n- Dropping `time`: Removes redundant columns after transforming them into meaningful features.\r\n","metadata":{}},{"cell_type":"code","source":"# Ensure 'time' is in datetime format\ntrain['time'] = pd.to_datetime(train['time'], errors='coerce')# Convert to datetime, invalid entries become NaT\ntest['time'] = pd.to_datetime(test['time'], errors='coerce')  # Apply to test as well\n\n# Now you can safely use the .dt accessor\ntrain[\"sin_hour\"] = np.sin(np.pi * train['time'].dt.hour / 12)\ntrain[\"cos_hour\"] = np.cos(np.pi * train['time'].dt.hour / 12)\n\ntest[\"sin_hour\"] = np.sin(np.pi * test['time'].dt.hour / 12)\ntest[\"cos_hour\"] = np.cos(np.pi * test['time'].dt.hour / 12)\n\n\ntrain = train.drop('time' , axis = 1)\ntest = test.drop('time' , axis = 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:31.095209Z","iopub.execute_input":"2024-11-29T21:15:31.095675Z","iopub.status.idle":"2024-11-29T21:15:31.547403Z","shell.execute_reply.started":"2024-11-29T21:15:31.095628Z","shell.execute_reply":"2024-11-29T21:15:31.546251Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Preprocessing Summary\r\n\r\n### **1. Dropping Columns with High Percentage of Missing Values**  \r\nA function `drop_columns_with_high_nan` is implemented to drop columns from the dataset that have more than a specified threshold (default 25%) of missing values. This step helps in cleaning the dataset by removing columns that are largely incomplete and may not contribute meaningfully to the model.\r\n\r\n### **2. Function Details:**\r\n- **Input:** \r\n    - The function takes the input DataFrame (`df`) and a threshold value to determine the percentage of missing values.\r\n    - Default threshold is set to 25% (0.25).\r\n- **Output:** \r\n    - It returns two objects: the cleaned DataFrame (with the problematic columns dropped) and a list of the column names that were dropped due to having too many missing value\n\r\n### **3. Purpose:**\r\n- Removing columns with a high percentage of missing values ensures that the model does not learn from incomplete or irrelevant data, improving overall performance and reducing noise.\r\n- The `threshold` parameter can be adjusted depending on how strict you want to be regarding missing data.\r\n","metadata":{}},{"cell_type":"code","source":"def drop_columns_with_high_nan(df, threshold=0.5):\n    \"\"\"\n    Drops columns from the DataFrame if they have more than a specified\n    percentage of missing values and returns the names of these columns.\n\n    Parameters:\n        df (pd.DataFrame): The input DataFrame.\n        threshold (float): The percentage threshold for missing values (default is 50%).\n\n    Returns:\n        tuple: \n            - pd.DataFrame: The DataFrame with columns having more than `threshold`% missing values dropped.\n            - list: Names of the columns that were dropped.\n    \"\"\"\n    # Calculate the threshold count for missing values\n    missing_threshold = len(df) * threshold\n\n    # Identify columns with more missing values than the threshold\n    high_nan_columns = df.columns[df.isnull().sum() > missing_threshold].tolist()\n\n    # Drop the identified columns\n    cleaned_df = df.drop(columns=high_nan_columns)\n\n    return cleaned_df, high_nan_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:31.548615Z","iopub.execute_input":"2024-11-29T21:15:31.548970Z","iopub.status.idle":"2024-11-29T21:15:31.555415Z","shell.execute_reply.started":"2024-11-29T21:15:31.548924Z","shell.execute_reply":"2024-11-29T21:15:31.554344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the function\ntrain_cleaned, dropped_columns = drop_columns_with_high_nan(train, threshold=0.5)\n\n\nprint(\"\\nDropped Columns:\")\nprint(dropped_columns)\n\nprint(\"Cleaned DataFrame:\")\ntrain_cleaned.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:31.556950Z","iopub.execute_input":"2024-11-29T21:15:31.557331Z","iopub.status.idle":"2024-11-29T21:15:32.222881Z","shell.execute_reply.started":"2024-11-29T21:15:31.557285Z","shell.execute_reply":"2024-11-29T21:15:32.221619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Preprocessing Summary\n\n### **Applying Column Removal from Train to Test Data**  \nThe function `apply_column_removal_to_test` is designed to remove columns with a high percentage of missing values from the test data, based on the columns removed from the training data. This ensures consistency between the train and test datasets by keeping only the relevant features.\n\n\n### **Purpose:**\n- This step ensures that the test data remains consistent with the train data by applying the same column removal logic to both datasets.\n- The dropped columns are those that have more than the specified threshold percentage of missing values in the training data. These columns are also removed from the test set to maintain feature alignment.\n\n","metadata":{}},{"cell_type":"code","source":"def apply_column_removal_to_test(train_df, test_df, threshold=0.5):\n    \"\"\"\n    Applies column removal based on NaN threshold from train data to test data.\n    \n    Parameters:\n        train_df (pd.DataFrame): The training DataFrame.\n        test_df (pd.DataFrame): The test DataFrame.\n        threshold (float): The percentage threshold for missing values (default is 50%).\n\n    Returns:\n        tuple:\n            - pd.DataFrame: Cleaned test DataFrame with columns dropped.\n            - list: Names of the columns dropped from the test data.\n    \"\"\"\n    # Drop columns in train data and get the names of dropped columns\n    _, high_nan_columns_train = drop_columns_with_high_nan(train_df, threshold)\n\n    # Remove the same columns from test data\n    cleaned_test_df = test_df.drop(columns=high_nan_columns_train, errors='ignore')\n\n    return cleaned_test_df, high_nan_columns_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:32.224381Z","iopub.execute_input":"2024-11-29T21:15:32.224817Z","iopub.status.idle":"2024-11-29T21:15:32.231459Z","shell.execute_reply.started":"2024-11-29T21:15:32.224756Z","shell.execute_reply":"2024-11-29T21:15:32.230389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the function\ntest_cleaned, dropped_columns = apply_column_removal_to_test(train, test, threshold=0.5)\n\n\n\nprint(\"\\nDropped Columns from Test Data:\")\nprint(dropped_columns)\n\nprint(\"Cleaned Test DataFrame:\")\ntest_cleaned.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:32.232661Z","iopub.execute_input":"2024-11-29T21:15:32.233071Z","iopub.status.idle":"2024-11-29T21:15:32.879472Z","shell.execute_reply.started":"2024-11-29T21:15:32.233036Z","shell.execute_reply":"2024-11-29T21:15:32.878373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Updated df_train:\",train_cleaned.shape)\nprint(\"Updated df_test:\",test_cleaned.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:32.883255Z","iopub.execute_input":"2024-11-29T21:15:32.883622Z","iopub.status.idle":"2024-11-29T21:15:32.889705Z","shell.execute_reply.started":"2024-11-29T21:15:32.883586Z","shell.execute_reply":"2024-11-29T21:15:32.888446Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing Summary \r\n\r\n## 1. Handling Missing Values for Different Columns\r\nThe updated code handles missing values based on column types and their characteristics. Blood glucose (`bg-`) and heart rate (`hr`) columns are processed using **interpolation**, while columns related to `insulin`, `calories`, `steps`, and `carbs` are filled with `0`.\r\n\r\n---\r\n\r\n## 2. Functionality Breakdown\r\n\r\n### For Blood Glucose (`bg-`) and Heart Rate (`hr`) Columns:\r\n- **Missing values are filled using linear interpolation**. This method estimates missing values based on their neighboring observations within a row.\r\n- Interpolation ensures a smooth and consistent representation of data trends, preserving logical continuity.\r\n- **Interpolation is applied across rows (`axis=1`)**, meaning missing values are calculated horizontally for each individual data entry.\r\n- This approach maintains the trend and avoids abrupt or unrealistic jumps in blood glucose and heart rate values.\r\n\r\n---\r\n\r\n### For Insulin, Calories, Steps, and Carbs Columns (`insulin`, `cals`, `steps`, `carbs`):\r\n- **Missing values in these columns are replaced with `0`.**\r\n- Filling with `0` reflects the assumption that missing data corresponds to no recorded activity, intake, or administration (e.g., no insulin or calorie intake).\r\n- **Columns included:**\r\n  - `insulin_cols`: Columns related to insulin levels.\r\n  - `cals_cols`: Columns representing calorie data.\r\n  - `steps_cols`: Columns containing step count data.\r\n  - `carbs_cols`: Columns related to carbohydrate intake.\r\n\r\n---\r\n\r\n## 3. Purpose\r\n\r\n### Blood Glucose and Heart Rate Data:\r\n- Interpolation ensures that the data for `bg-` and `hr` columns is complete and continuous, suitable for analysis of trends or patterns over time.\r\n- This method is especially appropriate for time-series or physiological data, where missing values are best estimated based on adjacent readings.\r\n\r\n### Insulin, Calories, Steps, and Carbs Data:\r\n- Filling with `0` ensures that these columns are ready for analysis without introducing artificial trends.\r\n- The assumption of `0` for missing values simplifies preprocessing and is appropriate for contexts where the absence of data implies no corresponding activity or measurement.\r\nfic trend or pattern.\r\n","metadata":{}},{"cell_type":"code","source":"# Define related column sets (bg_cols, insulin_cols, cals_cols, steps_cols, carbs_cols, hr_cols)\nbg_cols = [col for col in train_cleaned.columns if col.startswith('bg-')]\nhr_cols = [col for col in train_cleaned.columns if col.startswith('hr')]\n\ncarbs_cols = [col for col in test_cleaned.columns if col.startswith('carbs')]\ninsulin_cols = [col for col in test_cleaned.columns if col.startswith('insulin')]\nsteps_cols = [col for col in test_cleaned.columns if col.startswith('steps')]\ncals_cols = [col for col in test_cleaned.columns if col.startswith('cals')]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:32.890732Z","iopub.execute_input":"2024-11-29T21:15:32.891098Z","iopub.status.idle":"2024-11-29T21:15:32.906159Z","shell.execute_reply.started":"2024-11-29T21:15:32.891063Z","shell.execute_reply":"2024-11-29T21:15:32.904673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine all relevant columns\nfill_zero_cols = carbs_cols + insulin_cols + steps_cols + cals_cols\n\n# Fill missing values with 0 for these columns\ntrain_cleaned[fill_zero_cols] = train_cleaned[fill_zero_cols].fillna(0)\ntest_cleaned[fill_zero_cols] = test_cleaned[fill_zero_cols].fillna(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:32.907588Z","iopub.execute_input":"2024-11-29T21:15:32.907992Z","iopub.status.idle":"2024-11-29T21:15:33.398330Z","shell.execute_reply.started":"2024-11-29T21:15:32.907950Z","shell.execute_reply":"2024-11-29T21:15:33.397124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = bg_cols + hr_cols\n\n\n\n# Step 1: Linear interpolation for missing values in bg_cols and hr_cols\ntrain_cleaned[time_series_cols] = train_cleaned[time_series_cols].interpolate(\n    method='linear', axis=1, limit_direction='both')\ntest_cleaned[time_series_cols] = test_cleaned[time_series_cols].interpolate(\n    method='linear', axis=1, limit_direction='both')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:33.399681Z","iopub.execute_input":"2024-11-29T21:15:33.400145Z","iopub.status.idle":"2024-11-29T21:15:39.920982Z","shell.execute_reply.started":"2024-11-29T21:15:33.400094Z","shell.execute_reply":"2024-11-29T21:15:39.919731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Checking for Remaining Missing Values\n\nAfter applying the missing value imputation strategies, we check if there are any remaining missing values in the `train_cleaned` and `test_cleaned` datasets.\n\n## **Purpose of the Check:**\n### **Objective:** \nThe check ensures that all missing values have been handled effectively during the preprocessing steps.\n\n### **Expected Result:** \nIf no missing values remain, both outputs should print False, indicating that there are no missing values in the datasets.\n\n### **If True:** \nIf True is returned, there are still columns with missing values in the dataset, requiring further imputation or investigation.","metadata":{}},{"cell_type":"code","source":"\nprint(train_cleaned.isnull().sum().any())\n\nprint(test_cleaned.isnull().sum().any())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:39.922390Z","iopub.execute_input":"2024-11-29T21:15:39.922942Z","iopub.status.idle":"2024-11-29T21:15:40.064100Z","shell.execute_reply.started":"2024-11-29T21:15:39.922869Z","shell.execute_reply":"2024-11-29T21:15:40.062855Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Extracting Specific Columns for Analysis\r\n\r\n### **2. Purpose:**\r\n\r\n#### **Identifying Relevant Columns:**\r\n\r\n- **`past_hour_columns`:** Extracts columns that contain the prefix `\"-0:\"`, which likely represent the previous hour's data.\r\n  \r\n- **`columns_to_keep`:** Combines important features including:\r\n  - **`p_num`:** Participant number (numeric).\r\n  - **`sin_hour` & `cos_hour`:** Features representing the time of day, encoded using sine and cosine transformations to capture cyclical patterns in time.\r\n  - **`bg+1:00`:** The target column, which likely represents the glucose measurement for the next hour.\r\n  - **Past Hour Columns:** The columns that represent data from the previous hour(s), identified by the prefix `\"-0:\"`.\rOne Hour DataOne Hour Ahead?**\r\n\r\nThe goal is to predict the glucose values one hour ahead. By including features such as the sine and cosine transformations of the hour, along with the data from the previous hour, the model can capture both the cyclical nature of time and the immediate past behavior of glucose measurements. Using data from just the past hour is sufficient to capture meaningful patterns and trends that help predict the next value. This approach keeps the model simple while effectively predicting the target.\r\n\r\n### **3. Resulting DataFrame:**\r\n\r\nThe filtered DataFrame `train_df` now includes only the relevant columns for further analysis or model training. It contains the necessary information from the previous hour, participant number, time of day, and the target glucose measurement for the next hour.\r\nmodel training.\r\n","metadata":{}},{"cell_type":"code","source":"# Extract past hour columns (prefix \"0:\")\npast_hour_columns = [col for col in train_cleaned.columns if \"-0:\" in col]\n\n# Include participant number (p_num), sin_hour, cos_hour, and target column (bg+1:00)\ncolumns_to_keep = ['p_num', 'sin_hour', 'cos_hour', 'bg+1:00'] + past_hour_columns\n\n# Filter the DataFrame\ntrain_df = train_cleaned[columns_to_keep]\n\n# Display the result\ntrain_df.sample(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.065705Z","iopub.execute_input":"2024-11-29T21:15:40.066637Z","iopub.status.idle":"2024-11-29T21:15:40.151008Z","shell.execute_reply.started":"2024-11-29T21:15:40.066579Z","shell.execute_reply":"2024-11-29T21:15:40.149699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract past hour columns (prefix \"0:\")\npast_hour_columns = [col for col in test_cleaned.columns if \"-0:\" in col]\n\n# Include participant number (p_num), sin_hour, cos_hour, and target column (bg+1:00)\ncolumns_to_keep_in_test = ['p_num', 'sin_hour', 'cos_hour'] + past_hour_columns\n\n# Filter the DataFrame\ntest_df = test_cleaned[columns_to_keep_in_test]\n\n# Display the result\ntest_df.sample(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.152976Z","iopub.execute_input":"2024-11-29T21:15:40.153300Z","iopub.status.idle":"2024-11-29T21:15:40.183563Z","shell.execute_reply.started":"2024-11-29T21:15:40.153269Z","shell.execute_reply":"2024-11-29T21:15:40.182442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Identifying Missing Columns Between Train and Test DataFrames\n\n### **Purpose:**\nBefore training a model, it's important to ensure that the train and test datasets have the same columns. This section of the code helps identify any columns that are missing from either the training or the test set. \n","metadata":{}},{"cell_type":"code","source":"# Identify missing columns in train compared to test\nmissing_in_train = set(test_df.columns) - set(train_df.columns)\nprint(f\"Missing columns in train: {missing_in_train}\")\n\n# Identify missing columns in test compared to train\nmissing_in_test = set(train_df.columns) - set(test_df.columns)\nprint(f\"Missing columns in test: {missing_in_test}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.184794Z","iopub.execute_input":"2024-11-29T21:15:40.185149Z","iopub.status.idle":"2024-11-29T21:15:40.192199Z","shell.execute_reply.started":"2024-11-29T21:15:40.185115Z","shell.execute_reply":"2024-11-29T21:15:40.190867Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Checking for Missing Values in Train and Test DataFrames\n\n### **Purpose:**\nIt is essential to verify whether there are missing values in both the training and test datasets before proceeding with model training. Missing values can affect model performance and should be handled appropriately.\n","metadata":{}},{"cell_type":"code","source":"# Check for missing values in train_filtered\nmissing_train = train_df.isnull().sum()\nprint(\"Missing values in train_filtered:\")\nprint(missing_train[missing_train > 0])\n\n# Check for missing values in test_df\nmissing_test = test_df.isnull().sum()\nprint(\"Missing values in test_df:\")\nprint(missing_test[missing_test > 0])\n\n# Summary\nif missing_train.sum() == 0 and missing_test.sum() == 0:\n    print(\"Both datasets are complete: No missing values.\")\nelse:\n    print(\"Missing values found. Please address them.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.193938Z","iopub.execute_input":"2024-11-29T21:15:40.194533Z","iopub.status.idle":"2024-11-29T21:15:40.231773Z","shell.execute_reply.started":"2024-11-29T21:15:40.194483Z","shell.execute_reply":"2024-11-29T21:15:40.230581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Renaming Columns for Consistency\n\n### **Purpose:**\nThe purpose of renaming the columns is to ensure consistency and avoid issues when working with certain characters that may not be compatible with some machine learning frameworks. In particular:\n- Special characters like `:` and `-` may cause issues in column names.\n- We keep the target column name unchanged to avoid any disruptions in target-related tasks.\n","metadata":{}},{"cell_type":"code","source":"# Define the target column name before renaming\ntarget_column = 'bg+1:00'\n\n# Sanitize only feature columns\ntrain_df = train_df.rename(columns=lambda col: col if col == target_column else col.replace(':', '_').replace('-', '_'))\ntest_df = test_df.rename(columns=lambda col: col if col == target_column else col.replace(':', '_').replace('-', '_'))\n\n# Verify that the target column name remains unchanged\nprint(\"Sanitized Train Columns:\", train_df.columns.tolist())\nprint(\"Sanitized Test Columns:\", test_df.columns.tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.233085Z","iopub.execute_input":"2024-11-29T21:15:40.233428Z","iopub.status.idle":"2024-11-29T21:15:40.353405Z","shell.execute_reply.started":"2024-11-29T21:15:40.233393Z","shell.execute_reply":"2024-11-29T21:15:40.352315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **Inspecting the Shape of the DataFrames**\n\nTo understand the structure of the datasets, we print the shape of both the training and test DataFrames. The shape provides important information on the number of rows and columns in each dataset, helping us assess the size and completeness of the data before proceeding with any further analysis or modeling.\n","metadata":{}},{"cell_type":"code","source":"# Inspect the overall shape of the DataFrames\nprint(\"Overall Shapes:\")\nprint(f\"Train DataFrame Shape: {train_df.shape}\")\nprint(f\"Test DataFrame Shape: {test_df.shape}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.388471Z","iopub.execute_input":"2024-11-29T21:15:40.388791Z","iopub.status.idle":"2024-11-29T21:15:40.409832Z","shell.execute_reply.started":"2024-11-29T21:15:40.388757Z","shell.execute_reply":"2024-11-29T21:15:40.408656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import skew \n\n# Plot histogram\nplt.figure(figsize=(8, 6))\nsns.histplot(train_df['bg+1:00'], kde=True, color='blue')\nplt.title('Distribution of Target Variable (y)')\nplt.xlabel('Target Variable (y)')\nplt.ylabel('Frequency')\nplt.show()\n\n# Calculate skewness\nskewness_value = skew(train_df['bg+1:00'])\nprint(f\"Skewness of Target Variable (y) before transformation: {skewness_value:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:40.411277Z","iopub.execute_input":"2024-11-29T21:15:40.411619Z","iopub.status.idle":"2024-11-29T21:15:42.216307Z","shell.execute_reply.started":"2024-11-29T21:15:40.411585Z","shell.execute_reply":"2024-11-29T21:15:42.215139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Log transformation\ntrain_df['bg+1:00'] = np.log1p(train_df['bg+1:00'])\n\n# Plot histogram of transformed target\nsns.histplot(train_df['bg+1:00'], kde=True, color='green')\nplt.title('Distribution of Log-Transformed Target Variable (y_log)')\nplt.show()\n\n# Check skewness after transformation\nlog_skewness_value = skew(train_df['bg+1:00'])\nprint(f\"Skewness after Log Transformation: {log_skewness_value:.2f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:42.217481Z","iopub.execute_input":"2024-11-29T21:15:42.218012Z","iopub.status.idle":"2024-11-29T21:15:43.574870Z","shell.execute_reply.started":"2024-11-29T21:15:42.217969Z","shell.execute_reply":"2024-11-29T21:15:43.573768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **Splitting Data into Features and Target**\n\nThe first step in preparing the data for machine learning is separating the target variable (`bg+1:00`) from the features. The target variable represents the blood glucose measurement for the next hour, which we aim to predict. The remaining columns are used as the input features for the model.\n\n#### **Code Explanation:**\n\n1. **Separating Features and Target:**\n   - `X`: This is the feature matrix created by dropping the target column (`bg+1:00`) and the participant number column (`p_num`) from the DataFrame.\n   - `y`: This is the target vector, which contains the values from the `bg+1:00` column.\n\n2. **Splitting the Data into Training and Validation Sets:**\n   - The dataset is split into training and validation sets using `train_test_split` from `sklearn.model_selection`.\n   - The `test_size=0.2` argument indicates that 20% of the data will be used for validation, while the remaining 80% will be used for training.\n   - `random_state=42` ensures reproducibility of the split.","metadata":{}},{"cell_type":"code","source":"X = train_df.drop(['bg+1:00', 'p_num'], axis=1)\ny = train_df['bg+1:00']\ngroups = train_df['p_num']\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2 , random_state = 42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:43.576239Z","iopub.execute_input":"2024-11-29T21:15:43.576564Z","iopub.status.idle":"2024-11-29T21:15:43.681086Z","shell.execute_reply.started":"2024-11-29T21:15:43.576530Z","shell.execute_reply":"2024-11-29T21:15:43.679980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\n\n# Initialize RobustScaler\nscaler = RobustScaler()\n\n# Apply the scaler to the features\nX_train_scaled = scaler.fit_transform(X_train)\nX_val_scaled = scaler.transform(X_val)\n\n# Verify the scaling\nX_train_scaled[:2]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:43.682461Z","iopub.execute_input":"2024-11-29T21:15:43.682816Z","iopub.status.idle":"2024-11-29T21:15:44.062793Z","shell.execute_reply.started":"2024-11-29T21:15:43.682780Z","shell.execute_reply":"2024-11-29T21:15:44.061574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:44.064062Z","iopub.execute_input":"2024-11-29T21:15:44.064400Z","iopub.status.idle":"2024-11-29T21:15:44.073230Z","shell.execute_reply.started":"2024-11-29T21:15:44.064364Z","shell.execute_reply":"2024-11-29T21:15:44.072076Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **Inspecting the Shape of the Training and Validation Data**\n\nAfter splitting the dataset into features (`X`) and target (`y`), and further dividing them into training and validation sets, it's important to verify the shape of the resulting datasets. This ensures that the split was performed correctly and that we have the expected number of samples in both sets.\n\n#### **Code Explanation:**\n\nThe following code prints the shape of the training and validation data:\n\n1. **Training Data (`X_train`):** This will contain 80% of the data for model training.\n2. **Validation Data (`X_val`):** This will contain the remaining 20% of the data, which will be used for model evaluation.\n","metadata":{}},{"cell_type":"code","source":"\n\nprint(f\"Training data shape: {X_train.shape}\")\nprint(f\"Validation data shape: {X_val.shape}\")\n\n\n\n\nprint(f\"Training data shape after scaling: {X_train_scaled.shape}\")\nprint(f\"Validation data shapeafter scaling: {X_val_scaled.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:44.078595Z","iopub.execute_input":"2024-11-29T21:15:44.079018Z","iopub.status.idle":"2024-11-29T21:15:44.088331Z","shell.execute_reply.started":"2024-11-29T21:15:44.078979Z","shell.execute_reply":"2024-11-29T21:15:44.086963Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"-------","metadata":{}},{"cell_type":"code","source":"from fastai.tabular.all import *\nimport pandas as pd\nfrom fastai.callback.tracker import EarlyStoppingCallback, SaveModelCallback","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:44.089495Z","iopub.execute_input":"2024-11-29T21:15:44.089987Z","iopub.status.idle":"2024-11-29T21:15:47.851090Z","shell.execute_reply.started":"2024-11-29T21:15:44.089941Z","shell.execute_reply":"2024-11-29T21:15:47.849882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.tabular.all import TabularDataLoaders, TabularLearner, accuracy, rmse, Categorify, FillMissing, Normalize\nimport pandas as pd\nimport numpy as np\n\n# Convert scaled features back to DataFrame for FastAI\nX_train_scaled_df = pd.DataFrame(X_train_scaled, columns=X_train.columns)\nX_val_scaled_df = pd.DataFrame(X_val_scaled, columns = X_val.columns)\n\n# Combine scaled features with their corresponding target\ntrain_scaled = X_train_scaled_df.copy()\ntrain_scaled['bg+1:00'] = y_train.values  # Replace 'target' with your actual target column name\n\nval_scaled = X_val_scaled_df.copy()\nval_scaled['bg+1:00'] = y_val.values  # Replace 'target' with your actual target column name\n\n# Get continuous features in one line, excluding 'bg+1:00' (or use your actual target column name)\ncont_features = [col for col in train_scaled.select_dtypes(include=['int64', 'float64', 'int32', 'float32'])\n                 .columns if col != 'bg+1:00']\n\n# Create TabularDataLoaders for FastAI using the combined train and validation sets\ndls = TabularDataLoaders.from_df(\n    train_scaled,  # Use the pre-scaled train set\n    procs=[Categorify],  # Add processors to handle missing values and normalization\n    cont_names=cont_features,  # Pass in the continuous feature names\n    y_names='bg+1:00',  # Pass in the target column name;\n    valid_idx=val_scaled.index  # Specify validation indices\n)\n\n# Check the DataLoaders to ensure everything looks good\nprint(dls.train.dataset)  # Inspect the first row in the training set\nprint(dls.valid.dataset)  # Inspect the first row in the validation set","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:47.852422Z","iopub.execute_input":"2024-11-29T21:15:47.852971Z","iopub.status.idle":"2024-11-29T21:15:48.326969Z","shell.execute_reply.started":"2024-11-29T21:15:47.852924Z","shell.execute_reply":"2024-11-29T21:15:48.325825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.tabular.all import *\n\nconfig = tabular_config( ps=0.4) # Dropout rates for each layer\n\n# Create a TabularLearner with custom configuration\nlearn = tabular_learner(\n    dls, \n    layers=[2000, 1500, 1000, 500, 300, 100],\n    metrics=rmse,         # Evaluation metric\n    config=config,        # Configuration for dropout and other parameters\n    cbs=[                 # Callbacks for early stopping and saving the best model\n        EarlyStoppingCallback(monitor='valid_loss', patience=5),\n        SaveModelCallback(monitor='valid_loss', fname='best_model')\n    ]\n)\n\n# Print the model to verify its structure\nprint(learn)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:48.328307Z","iopub.execute_input":"2024-11-29T21:15:48.328659Z","iopub.status.idle":"2024-11-29T21:15:48.405909Z","shell.execute_reply.started":"2024-11-29T21:15:48.328625Z","shell.execute_reply":"2024-11-29T21:15:48.404687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Finding optimal learning rate\n#lr = learn.lr_find()\nlr = learn.lr_find(start_lr=1e-6, end_lr=1e-1)\n\nprint(lr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:48.407412Z","iopub.execute_input":"2024-11-29T21:15:48.407864Z","iopub.status.idle":"2024-11-29T21:15:54.596983Z","shell.execute_reply.started":"2024-11-29T21:15:48.407811Z","shell.execute_reply":"2024-11-29T21:15:54.595954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training with one-cycle policy\nlearn.fit_one_cycle(100, lr_max = 0.01)# wd=1e-4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:15:54.598464Z","iopub.execute_input":"2024-11-29T21:15:54.599082Z","iopub.status.idle":"2024-11-29T21:40:51.467315Z","shell.execute_reply.started":"2024-11-29T21:15:54.599028Z","shell.execute_reply":"2024-11-29T21:40:51.466260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the best model saved during training\nlearn.load('best_model')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:40:51.468762Z","iopub.execute_input":"2024-11-29T21:40:51.469271Z","iopub.status.idle":"2024-11-29T21:40:51.494577Z","shell.execute_reply.started":"2024-11-29T21:40:51.469214Z","shell.execute_reply":"2024-11-29T21:40:51.493420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learn.recorder.plot_loss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:40:51.496066Z","iopub.execute_input":"2024-11-29T21:40:51.496509Z","iopub.status.idle":"2024-11-29T21:40:52.539570Z","shell.execute_reply.started":"2024-11-29T21:40:51.496462Z","shell.execute_reply":"2024-11-29T21:40:52.538274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get predictions on the validation set\npreds, targs = learn.get_preds(ds_idx=1)  # ds_idx=1 specifies the validation set in FastAI\n\n# Convert predictions and targets to numpy arrays\ny_pred = preds.numpy()\ny_val = targs.numpy()\n\n\n# Assuming y_pred and y_val were log-transformed before the split\ny_pred_original_scale = np.expm1(y_pred)  # Reverse log transformation on predictions\ny_val_original_scale = np.expm1(y_val)    # Reverse log transformation on actual target values\n\n# Optional: Print or check the shapes to ensure correctness\nprint(\"Predictions on original scale:\", y_pred_original_scale.shape)\nprint(\"Targets on original scale:\", y_val_original_scale.shape)\n\n# If you want to check the first few predictions and targets on the original scale:\nprint(\"First 5 predictions (original scale):\", y_pred_original_scale[:5])\nprint(\"First 5 targets (original scale):\", y_val_original_scale[:5])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:40:52.541084Z","iopub.execute_input":"2024-11-29T21:40:52.541542Z","iopub.status.idle":"2024-11-29T21:40:59.965682Z","shell.execute_reply.started":"2024-11-29T21:40:52.541491Z","shell.execute_reply":"2024-11-29T21:40:59.964560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nimport numpy as np\n\n# Calculate RMSE, MAE, and R²\n\n# Calculate RMSE, MAE, and R² on the original scale\nrmse = np.sqrt(mean_squared_error(y_val_original_scale, y_pred_original_scale))\nmae = mean_absolute_error(y_val_original_scale, y_pred_original_scale)\nr2 = r2_score(y_val_original_scale, y_pred_original_scale)\n\n# Print out the results\nprint(f\"RMSE: {rmse}\")\nprint(f\"MAE: {mae}\")\nprint(f\"R²: {r2}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:40:59.967384Z","iopub.execute_input":"2024-11-29T21:40:59.967858Z","iopub.status.idle":"2024-11-29T21:40:59.978592Z","shell.execute_reply.started":"2024-11-29T21:40:59.967788Z","shell.execute_reply":"2024-11-29T21:40:59.977264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\nfrom sklearn.metrics import mean_squared_error\n\n# Calculate residuals\nresiduals = y_val_original_scale.flatten() - y_pred_original_scale.flatten()\n\n# Plot the Q-Q plot\nplt.figure(figsize=(8, 6))\nstats.probplot(residuals, dist=\"norm\", plot=plt)\nplt.title(\"Q-Q Plot of Residuals for Stacking Model\")\nplt.xlabel(\"Theoretical Quantiles\")\nplt.ylabel(\"Sample Quantiles\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:40:59.980154Z","iopub.execute_input":"2024-11-29T21:40:59.980539Z","iopub.status.idle":"2024-11-29T21:41:00.335447Z","shell.execute_reply.started":"2024-11-29T21:40:59.980503Z","shell.execute_reply":"2024-11-29T21:41:00.334315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate residuals\nresiduals = y_val_original_scale.flatten() - y_pred_original_scale.flatten()\n# Plot the residuals\nplt.figure(figsize=(10, 6))\nplt.scatter(y_val_original_scale, residuals, alpha=0.5, color='green')\nplt.axhline(y=0, color='red', linestyle='--')  # Add a horizontal line at 0\nplt.title(\"Residuals Plot\")\nplt.xlabel(\"Actual Values\")\nplt.ylabel(\"Residuals\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:41:00.336806Z","iopub.execute_input":"2024-11-29T21:41:00.337190Z","iopub.status.idle":"2024-11-29T21:41:00.716455Z","shell.execute_reply.started":"2024-11-29T21:41:00.337155Z","shell.execute_reply":"2024-11-29T21:41:00.715290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Plot the Actual vs Predicted Glucose Levels\nplt.figure(figsize=(12, 6))\nplt.plot(y_val_original_scale, label='Actual Glucose Levels', color='blue', linestyle='-', linewidth=1)\nplt.plot(y_pred_original_scale, label='Predicted Glucose Levels', color='green', linestyle='--', linewidth=1)\nplt.xlabel('Samples')\nplt.ylabel('Glucose Level')\nplt.title('Predicted vs Actual Glucose Levels')\nplt.legend()\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:41:00.718073Z","iopub.execute_input":"2024-11-29T21:41:00.718427Z","iopub.status.idle":"2024-11-29T21:41:01.559298Z","shell.execute_reply.started":"2024-11-29T21:41:00.718391Z","shell.execute_reply":"2024-11-29T21:41:01.557864Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## interference\n","metadata":{}},{"cell_type":"code","source":"# Assuming 'scaler' is the StandardScaler fitted on training data\ntest_scaled = scaler.transform(test_df[cont_features])  # Ensure the same features are selected\n\n# Convert the scaled test data back to a DataFrame\ntest_scaled_df = pd.DataFrame(test_scaled, columns=cont_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:41:01.560878Z","iopub.execute_input":"2024-11-29T21:41:01.561304Z","iopub.status.idle":"2024-11-29T21:41:01.571948Z","shell.execute_reply.started":"2024-11-29T21:41:01.561266Z","shell.execute_reply":"2024-11-29T21:41:01.570970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare the Test DataLoader\ntest_dl = dls.test_dl(test_scaled_df)  # You already have 'dls' loaded from the training set\n\n# Load the best model\nlearn.load('best_model')\n\n# Make predictions on the test set\npreds, _ = learn.get_preds(dl=test_dl)\n\n# Convert predictions to a numpy array\npredictions = preds.numpy()\n\n# Reverse log transformation \n\npredictions_original = np.exp(predictions)  \n\n# Convert the predictions to a DataFrame\npred_df = pd.DataFrame(predictions_original, columns=['bg+1:00'])  # Adjust the column name if needed\n\n# Optionally, include the 'id' column in the predictions dataframe (from test set)\n# Assuming `sample` is your test set dataframe that has the 'id' column\npred_df['id'] = sample['id']  # Make sure to use the correct 'id' column from your test dataframe\n\n# Reorder columns to have 'id' first\npred_df = pred_df[['id', 'bg+1:00']]\n\n# Save predictions to a CSV file\npred_df.to_csv('submission.csv', index=False)\n\n# Optionally, print a message confirming successful saving\nprint(\"Predictions saved to 'submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:41:01.573306Z","iopub.execute_input":"2024-11-29T21:41:01.573661Z","iopub.status.idle":"2024-11-29T21:41:02.318173Z","shell.execute_reply.started":"2024-11-29T21:41:01.573625Z","shell.execute_reply":"2024-11-29T21:41:02.316864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T21:41:02.319546Z","iopub.execute_input":"2024-11-29T21:41:02.320003Z","iopub.status.idle":"2024-11-29T21:41:02.332470Z","shell.execute_reply.started":"2024-11-29T21:41:02.319956Z","shell.execute_reply":"2024-11-29T21:41:02.331368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}