{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🔮 Key Insights from XGBoost - Modified Version of Vineet Gupta's Notebook (Public Score: 2.5200 on 2024-11-19)\nOriginal author: Vineet Gupta\n\nOriginal notebook: https://www.kaggle.com/code/vineetgupta2023/key-insights-from-xgboost-score-2-200/notebook\n\nMany great insights appear in Vineet's notebook!\n\nAdded feature engineering in this notebook include:\n 1. Rolling statistics (mean and std) for blood glucose values\n 2. Lag features for insulin values\n 3. Rate of Change features\n 4. p_num specific features => look for .groupby('p_num')","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom IPython.display import HTML, Image\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Display header image and title\ndisplay(Image(url='https://upload.wikimedia.org/wikipedia/commons/thumb/7/7c/Kaggle_logo.png/400px-Kaggle_logo.png', width=100))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:05.974606Z","iopub.execute_input":"2024-11-26T20:14:05.975044Z","iopub.status.idle":"2024-11-26T20:14:09.120406Z","shell.execute_reply.started":"2024-11-26T20:14:05.975004Z","shell.execute_reply":"2024-11-26T20:14:09.119060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\"\"\n    📊 Blood Glucose Prediction\n    ===========================\n    🩺 Medical Time Series Analysis\n    📈 XGBoost Modeling\n    🔬 Feature Engineering\n    \n    Loading data...\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:09.122304Z","iopub.execute_input":"2024-11-26T20:14:09.122759Z","iopub.status.idle":"2024-11-26T20:14:09.129130Z","shell.execute_reply.started":"2024-11-26T20:14:09.122724Z","shell.execute_reply":"2024-11-26T20:14:09.127864Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/brist1d/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:09.130366Z","iopub.execute_input":"2024-11-26T20:14:09.130785Z","iopub.status.idle":"2024-11-26T20:14:22.022718Z","shell.execute_reply.started":"2024-11-26T20:14:09.130738Z","shell.execute_reply":"2024-11-26T20:14:22.021620Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Column Groups","metadata":{}},{"cell_type":"code","source":"bg_columns = [col for col in train.columns if col.startswith('bg-')]\ninsulin_columns = [col for col in train.columns if col.startswith('insulin-')]\nhr_columns = [col for col in train.columns if col.startswith('hr-')]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:22.025393Z","iopub.execute_input":"2024-11-26T20:14:22.025862Z","iopub.status.idle":"2024-11-26T20:14:22.032788Z","shell.execute_reply.started":"2024-11-26T20:14:22.025814Z","shell.execute_reply":"2024-11-26T20:14:22.031449Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Missing Values Handling","metadata":{}},{"cell_type":"code","source":"for df in [train, test]:\n    bg_columns = [col for col in df.columns if col.startswith('bg-')]\n    insulin_columns = [col for col in df.columns if col.startswith('insulin-')]\n    hr_columns = [col for col in df.columns if col.startswith('hr-')]\n\n    df['carbs-0:00'] = df.groupby('p_num')['carbs-0:00'].fillna(method='ffill').fillna(method='bfill')\n        \n    df[bg_columns] = df[bg_columns].ffill(axis=1).bfill(axis=1)\n    df[insulin_columns] = df[insulin_columns].ffill(axis=1).bfill(axis=1)\n    df[hr_columns] = df[hr_columns].ffill(axis=1).bfill(axis=1)\n    \n    df.fillna(-1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:22.034026Z","iopub.execute_input":"2024-11-26T20:14:22.034365Z","iopub.status.idle":"2024-11-26T20:14:25.523042Z","shell.execute_reply.started":"2024-11-26T20:14:22.034334Z","shell.execute_reply":"2024-11-26T20:14:25.521833Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering\n","metadata":{}},{"cell_type":"code","source":"def add_time_features(df):\n    df['hour'] = pd.to_datetime(df['time']).dt.hour\n    df['day_of_week'] = pd.to_datetime(df['time']).dt.dayofweek\n    df['hour_sin'] = np.sin(2 * np.pi * df['hour']/24)\n    df['hour_cos'] = np.cos(2 * np.pi * df['hour']/24)\n    return df\n    \ndef add_lag_features(df, columns, lag_periods):\n    base_columns = [col for col in columns if \n                   not any(suffix in col for suffix in ['_std', '_mean', '_min', '_max', '_roll', '_lag'])]\n    for col in base_columns:\n        for lag in lag_periods:\n            df[f'{col}_lag_{lag}'] = df.groupby('p_num')[col].shift(lag)        \n    return df\n\ndef add_rate_of_change_features(df):\n    # Only select base columns without any derived features or lags\n    bg_cols = sorted([col for col in df.columns \n                     if col.startswith('bg-') and \n                     len(col.split('_')) == 1 and  # No derived features with underscores\n                     ':' in col],  # Must have timestamp\n                    reverse=True)\n    \n    for i in range(len(bg_cols)-1):\n        col_current = bg_cols[i]\n        col_next = bg_cols[i+1]\n        \n        time_current = int(col_current.split('-')[1].split(':')[0]) * 60 + int(col_current.split('-')[1].split(':')[1])\n        time_next = int(col_next.split('-')[1].split(':')[0]) * 60 + int(col_next.split('-')[1].split(':')[1])\n        \n        time_diff = abs(time_next - time_current)\n        \n        if time_diff > 0:\n            df[f'bg_rate_{i}'] = (df[col_current] - df[col_next]) / time_diff\n        else:\n            df[f'bg_rate_{i}'] = 0\n            \n    rate_cols = [col for col in df.columns if 'bg_rate_' in col]\n    if len(rate_cols) >= 3:\n        df['bg_rate_mean_recent'] = df[rate_cols[:3]].mean(axis=1)\n        df['bg_rate_std_recent'] = df[rate_cols[:3]].std(axis=1)\n    \n    return df\n\ndef add_rolling_features(df, columns, windows):\n    # Convert numeric columns to float\n    base_columns = [col for col in columns if \n        not any(suffix in col for suffix in ['_std', '_mean', '_min', '_max', '_roll'])]\n    \n    for col in base_columns:\n        df[col] = pd.to_numeric(df[col], errors='coerce')\n        \n    for col in base_columns:\n        for window in windows:\n            df[f'{col}_roll_mean_{window}'] = (\n                df.groupby('p_num')[col]\n                .rolling(window=window, min_periods=1)\n                .mean()\n                .reset_index(0, drop=True)\n            )\n            df[f'{col}_roll_std_{window}'] = (\n                df.groupby('p_num')[col]\n                .rolling(window=window, min_periods=1)\n                .std()\n                .reset_index(0, drop=True)\n            )\n    return df\n\n\ndef add_participant_features(df):\n    participant_stats = df.groupby('p_num').agg({\n        'bg-0:00': ['mean', 'std'],\n        'insulin-0:00': ['mean', 'max'],\n        'carbs-0:00': ['mean', 'max']\n    })\n\n    # Rename columns without combining with p_num\n    participant_stats.columns = [f'participant_{col[0]}_{col[1]}' for col in participant_stats.columns]\n    return df.merge(participant_stats, on='p_num', how='left')\n    \nactivities = pd.DataFrame({'activity': ['Indoor climbing','Run','Strength training','Swim','Bike','Dancing',\n                                      'Stairclimber','Spinning','Walking','HIIT','Outdoor Bike','Walk',\n                                      'Aerobic Workout','Tennis','Workout','Hike','Zumba','Sport','Yoga',\n                                      'Swimming','Weights','Running']})\n\nunique_activities = {activity: i+1 for i, activity in enumerate(activities['activity'].unique())}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:46:27.442764Z","iopub.execute_input":"2024-11-26T20:46:27.444639Z","iopub.status.idle":"2024-11-26T20:46:27.468084Z","shell.execute_reply.started":"2024-11-26T20:46:27.444587Z","shell.execute_reply":"2024-11-26T20:46:27.466831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## EDA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:25.544526Z","iopub.execute_input":"2024-11-26T20:14:25.544836Z","iopub.status.idle":"2024-11-26T20:14:25.560172Z","shell.execute_reply.started":"2024-11-26T20:14:25.544804Z","shell.execute_reply":"2024-11-26T20:14:25.559121Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/brist1d/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T21:05:10.260906Z","iopub.execute_input":"2024-11-26T21:05:10.261900Z","iopub.status.idle":"2024-11-26T21:05:19.919262Z","shell.execute_reply.started":"2024-11-26T21:05:10.261858Z","shell.execute_reply":"2024-11-26T21:05:19.917865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display basic information about the dataset\nprint(\"Dataset shape:\", train.shape)\nprint(\"\\nColumn types:\")\nprint(train.dtypes.value_counts())\n\n# Display the first few rows of important columns\nimportant_columns = ['p_num', 'time', 'bg-0:00', 'insulin-0:00', 'carbs-0:00', 'hr-0:00', 'steps-0:00', 'cals-0:00', 'bg+1:00']\nprint(\"\\nFirst 5 rows of important columns:\")\nprint(train[important_columns].head())\n\n# Check for missing values\nprint(\"\\nMissing values in important columns:\")\nprint(train[important_columns].isnull().sum())\n\n# Display unique values in categorical columns\ncategorical_columns = ['p_num']\nfor col in categorical_columns:\n    print(f\"\\nUnique values in {col}:\")\n    print(train[col].value_counts())\n\n# Calculate basic statistics for numeric columns\nnumeric_columns = ['bg-0:00', 'insulin-0:00', 'carbs-0:00', 'hr-0:00', 'steps-0:00', 'cals-0:00', 'bg+1:00']\nprint(\"\\nBasic statistics for numeric columns:\")\nprint(train[numeric_columns].describe())\n\n# Check for correlations between features and target\ncorrelations = train[numeric_columns].corr()['bg+1:00'].sort_values(ascending=False)\nprint(\"\\nCorrelations with target (bg+1:00):\")\nprint(correlations)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:35.254232Z","iopub.execute_input":"2024-11-26T20:14:35.254562Z","iopub.status.idle":"2024-11-26T20:14:35.429449Z","shell.execute_reply.started":"2024-11-26T20:14:35.254528Z","shell.execute_reply":"2024-11-26T20:14:35.428160Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The Work","metadata":{}},{"cell_type":"code","source":"participants = pd.DataFrame({'participant': [f'p{str(i).zfill(2)}' for i in range(1, 16)]})\nunique_participant = {p: i+1 for i, p in enumerate(participants['participant'].unique())}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:48:35.853292Z","iopub.execute_input":"2024-11-26T20:48:35.854359Z","iopub.status.idle":"2024-11-26T20:48:35.860928Z","shell.execute_reply.started":"2024-11-26T20:48:35.854313Z","shell.execute_reply":"2024-11-26T20:48:35.859790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_unique_participants = train['p_num'].nunique()\nprint(f\"Number of unique participants: {num_unique_participants}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T21:05:52.928092Z","iopub.execute_input":"2024-11-26T21:05:52.928529Z","iopub.status.idle":"2024-11-26T21:05:52.942451Z","shell.execute_reply.started":"2024-11-26T21:05:52.928491Z","shell.execute_reply":"2024-11-26T21:05:52.941275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_encoders = {}\n\nfor df in [train, test]:    \n    \n    for col in df.columns:\n        if 'activity' in col:\n            df[col] = df[col].apply(lambda x: unique_activities.get(x, -1) if x != '' else -1)\n        if col == 'p_num':\n            df[col] = df[col].apply(lambda x: unique_participant.get(x, -1) if x != '' else -1)\n            \n    df.loc[df['insulin-0:00'] < 0, 'insulin-0:00'] = 0\n    \n    # 2. Add engineered features (assuming these functions are defined elsewhere)\n    df = add_time_features(df)\n    df = add_lag_features(df, bg_columns + insulin_columns, lag_periods=[12])\n    df = add_rate_of_change_features(df)\n    df = add_rolling_features(df, bg_columns, windows=[12, 72, 144])\n    df = add_rolling_features(df, insulin_columns, windows=[12, 72])\n    df = add_participant_features(df)\n    \n    # 3. Time-based feature engineering\n    df['hour'] = pd.to_datetime(df['time'], format='%H:%M:%S').dt.hour\n    df['minute'] = pd.to_datetime(df['time'], format='%H:%M:%S').dt.minute\n    df['hour_sin'] = np.sin(2 * np.pi * df['hour'] / 24)\n    df['hour_cos'] = np.cos(2 * np.pi * df['hour'] / 24)\n    df['minute_sin'] = np.sin(2 * np.pi * df['minute'] / 60)\n    df['minute_cos'] = np.cos(2 * np.pi * df['minute'] / 60)\n    \n    df.drop(['hour', 'minute', 'time'], axis=1, inplace=True)\n\n    # 4. Label encoding for categorical columns\n    categorical_columns = df.select_dtypes(include=['object']).columns\n\n    # Apply label encoding to categorical columns\n    for col in categorical_columns:\n        le = LabelEncoder()\n        df[col] = le.fit_transform(df[col].astype(str))    \n\n    # 5. Data Type Conversion and NaN Handling    \n    for col in df.columns:\n        if df[col].dtype == 'object':\n            df[col] = pd.to_numeric(df[col], errors='coerce')\n    \n    df.fillna(df.mean(), inplace=True)\n\n    # 6. Scaling numeric features\n    scaler = StandardScaler()\n    numeric_features = ['bg-0:00', 'insulin-0:00', 'carbs-0:00', 'hr-0:00', 'steps-0:00', 'cals-0:00']\n    df[numeric_features] = scaler.fit_transform(df[numeric_features])\n\n# Prepare features\nfeature_columns = [col for col in train.columns if col not in ['id', 'bg+1:00', 'time']]\nX = train[feature_columns]\ny = train['bg+1:00']\n\n# Initialize predictions collection\ntest_predictions = []\nval_scores = []\n\n# Group K-fold cross-validation\ngkf = GroupKFold(n_splits=3)\n\n# Train models and make predictions\nfor train_index, val_index in gkf.split(X, y, groups=train['p_num']):\n    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n    \n    model = XGBRegressor(\n        n_estimators=500,\n        learning_rate=0.01,\n        max_depth=6,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        reg_alpha=0.1,\n        reg_lambda=0.1,\n        random_state=42\n    )\n    \n    model.fit(\n        X_train, \n        y_train,\n        eval_set=[(X_val, y_val)],\n        early_stopping_rounds=50,\n        verbose=0\n    )\n    \n    val_pred = model.predict(X_val)\n    val_scores.append(np.sqrt(mean_squared_error(y_val, val_pred)))\n    \n    test_predictions.append(model.predict(test[feature_columns]))\n\n# Calculate final predictions and score\nfinal_predictions = np.mean(np.array(test_predictions), axis=0)\ncv_rmse = np.mean(val_scores)\n\nprint(f\"Cross-validation RMSE: {cv_rmse:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T21:05:59.644853Z","iopub.execute_input":"2024-11-26T21:05:59.645319Z","iopub.status.idle":"2024-11-26T21:17:12.506523Z","shell.execute_reply.started":"2024-11-26T21:05:59.645279Z","shell.execute_reply":"2024-11-26T21:17:12.504760Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Final feature importance","metadata":{}},{"cell_type":"code","source":"def analyze_feature_importance(model, feature_columns):\n    \"\"\"Print feature importance analysis\"\"\"\n    importances = pd.DataFrame({\n        'feature': feature_columns,\n        'importance': model.feature_importances_\n    })\n    \n    importances = importances.sort_values('importance', ascending=False)\n    importances['cumulative'] = importances['importance'].cumsum()\n    importances['importance_pct'] = importances['importance'] * 100\n    importances['cumulative_pct'] = importances['cumulative'] * 100\n    \n    print(\"\\nTop 15 Most Important Features:\")\n    print(importances.head(15).to_string(index=False))\n    \n    # Print feature type analysis\n    print(\"\\nFeature Type Analysis:\")\n    type_analysis = {\n        'bg_base': len([x for x in importances.head(15)['feature'] if 'bg-' in x and '_' not in x]),\n        'bg_rolling': len([x for x in importances.head(15)['feature'] if 'bg-' in x and 'roll' in x]),\n        'bg_rate': len([x for x in importances.head(15)['feature'] if 'bg_rate' in x]),\n        'participant': len([x for x in importances.head(15)['feature'] if 'p_' in x]),\n    }\n    for k, v in type_analysis.items():\n        print(f\"{k}: {v}\")\n    \n    return importances","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After the training loop, use the last trained model\nprint(\"\\nOverall Feature Importance Analysis:\")\nfinal_importances = analyze_feature_importance(model, feature_columns)\n\n# Save importances to DataFrame for further analysis if needed\nfeature_importance_df = pd.DataFrame({\n    'feature': feature_columns,\n    'importance': model.feature_importances_\n}).sort_values('importance', ascending=False)\n\nprint(\"\\nFeature importance statistics:\")\nprint(feature_importance_df.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:35.450251Z","iopub.status.idle":"2024-11-26T20:14:35.450648Z","shell.execute_reply.started":"2024-11-26T20:14:35.450468Z","shell.execute_reply":"2024-11-26T20:14:35.450489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display feature importance\nfeature_importance = pd.DataFrame({\n    'feature': X.columns,\n    'importance': model.feature_importances_\n}).sort_values('importance', ascending=False)\n\nplt.figure(figsize=(10, 6))\nsns.barplot(data=feature_importance.head(15), x='importance', y='feature')\nplt.title('Top 15 Most Important Features')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:35.452162Z","iopub.status.idle":"2024-11-26T20:14:35.452629Z","shell.execute_reply.started":"2024-11-26T20:14:35.452394Z","shell.execute_reply":"2024-11-26T20:14:35.452424Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## The Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'id': test['id'],\n    'bg+1:00': final_predictions\n})\n\n# Save submission\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T20:14:35.453683Z","iopub.status.idle":"2024-11-26T20:14:35.454057Z","shell.execute_reply.started":"2024-11-26T20:14:35.453887Z","shell.execute_reply":"2024-11-26T20:14:35.453907Z"}},"outputs":[],"execution_count":null}]}