{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Reader's note: This notebook serves as a baseline machine learning model for all the experiments that we are going to do in actual time series models (cnn 1d, RNN (GRU/LSTM), Transformer (Tiny or efficient variant)).","metadata":{}},{"cell_type":"markdown","source":"imports","metadata":{}},{"cell_type":"code","source":"#imports\nimport pandas as pd\nimport numpy as np\nimport random\nfrom lightgbm import LGBMClassifier\nimport joblib\nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\nfrom sklearn.model_selection import RandomizedSearchCV, train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom xgboost import XGBClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:42:11.862381Z","iopub.execute_input":"2025-06-08T22:42:11.863440Z","iopub.status.idle":"2025-06-08T22:42:12.386361Z","shell.execute_reply.started":"2025-06-08T22:42:11.863399Z","shell.execute_reply":"2025-06-08T22:42:12.384636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 1. Loading and Merging Datasets","metadata":{}},{"cell_type":"code","source":"# Load main data\ntrain_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ndemographics_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\n\n# Merge on 'subject'\nmerged_df = train_df.merge(demographics_df, on='subject', how='left')\n\n# Check the result\nprint(\"Merged shape:\", merged_df.shape)\nprint(merged_df[['subject', 'age', 'sex', 'gesture']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:12:30.034578Z","iopub.execute_input":"2025-06-08T22:12:30.034915Z","iopub.status.idle":"2025-06-08T22:13:06.538929Z","shell.execute_reply.started":"2025-06-08T22:12:30.034892Z","shell.execute_reply":"2025-06-08T22:13:06.537829Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2 : sequence level feture engineering","metadata":{}},{"cell_type":"code","source":"# Identify sensor columns\nimu_cols = [col for col in merged_df.columns if col.startswith('acc_') or col.startswith('rot_')]\nthm_cols = [col for col in merged_df.columns if col.startswith('thm_')]\ntof_cols = [col for col in merged_df.columns if col.startswith('tof_')]\n\n# Create a helper column: does this row have ToF data?\nmerged_df['has_tof_data'] = (merged_df[tof_cols] != -1).any(axis=1)\n\n# Group by sequence_id\ngrouped = merged_df.groupby('sequence_id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:13:25.397531Z","iopub.execute_input":"2025-06-08T22:13:25.397908Z","iopub.status.idle":"2025-06-08T22:13:26.351256Z","shell.execute_reply.started":"2025-06-08T22:13:25.397883Z","shell.execute_reply":"2025-06-08T22:13:26.350476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to extract stats\ndef extract_features(group):\n    features = {}\n    for col in imu_cols + thm_cols + tof_cols:\n        if col in group:\n            features[f'{col}_mean'] = group[col].mean()\n            features[f'{col}_std'] = group[col].std()\n            features[f'{col}_min'] = group[col].min()\n            features[f'{col}_max'] = group[col].max()\n    # Add gesture and subject-level info (demographics)\n    features['gesture'] = group['gesture'].iloc[0]\n    features['subject'] = group['subject'].iloc[0]\n    features['has_tof_data'] = group['has_tof_data'].any()\n    for demo_col in ['age', 'sex', 'handedness', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm', 'adult_child']:\n        features[demo_col] = group[demo_col].iloc[0]\n    return pd.Series(features)\n\n# Apply the feature extraction\nsequence_features_df = grouped.apply(extract_features).reset_index()\n\nprint(\"Final feature table shape:\", sequence_features_df.shape)\nprint(sequence_features_df[['sequence_id', 'gesture', 'has_tof_data']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:13:26.352404Z","iopub.execute_input":"2025-06-08T22:13:26.352714Z","iopub.status.idle":"2025-06-08T22:19:03.369993Z","shell.execute_reply.started":"2025-06-08T22:13:26.352691Z","shell.execute_reply":"2025-06-08T22:19:03.368881Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Inspection and Split","metadata":{}},{"cell_type":"code","source":"# 1. Missing value check\nmissing_counts = sequence_features_df.isnull().sum()\nprint(\"Missing values per column:\\n\", missing_counts[missing_counts > 0])\n\n# 2. Class distribution\nprint(\"\\nGesture distribution:\")\nprint(sequence_features_df['gesture'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:19:03.371667Z","iopub.execute_input":"2025-06-08T22:19:03.372027Z","iopub.status.idle":"2025-06-08T22:19:03.437584Z","shell.execute_reply.started":"2025-06-08T22:19:03.371995Z","shell.execute_reply":"2025-06-08T22:19:03.436682Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3a : simulate tof missing in half of training data","metadata":{}},{"cell_type":"code","source":"# Set seed for reproducibility\nrandom.seed(42)\n\n# Copy full dataset\nsequence_features_simulated = sequence_features_df.copy()\n\n# Identify all ToF feature columns\ntof_feature_cols = [col for col in sequence_features_df.columns if col.startswith('tof_')]\n\n# Randomly select 50% of sequence_ids\nsequence_ids = sequence_features_simulated['sequence_id'].unique().tolist()\nnum_to_mask = len(sequence_ids) // 2\nmasked_sequence_ids = set(random.sample(sequence_ids, num_to_mask))\n\n# Mask ToF features by setting to -1\nsequence_features_simulated.loc[\n    sequence_features_simulated['sequence_id'].isin(masked_sequence_ids),\n    tof_feature_cols\n] = -1\n\n# Update has_tof_data flag\nsequence_features_simulated['has_tof_data'] = ~sequence_features_simulated['sequence_id'].isin(masked_sequence_ids)\n\n# First, reserve holdout test set (e.g., 15%) from the full simulated data\ntrain_val_df, test_holdout_df = train_test_split(\n    sequence_features_simulated,\n    test_size=0.15,\n    stratify=sequence_features_simulated['gesture'],\n    random_state=42\n)\n\n\n# Create two sets\nwith_tof_df = train_val_df[train_val_df['has_tof_data']]\nwithout_tof_df = train_val_df[~train_val_df['has_tof_data']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:19:03.438688Z","iopub.execute_input":"2025-06-08T22:19:03.439169Z","iopub.status.idle":"2025-06-08T22:19:03.877809Z","shell.execute_reply.started":"2025-06-08T22:19:03.439134Z","shell.execute_reply":"2025-06-08T22:19:03.876623Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3b : stratified split","metadata":{}},{"cell_type":"code","source":"# For With-ToF data\ntrain_with_tof, val_with_tof = train_test_split(\n    with_tof_df,\n    test_size=0.2,\n    stratify=with_tof_df['gesture'],\n    random_state=42\n)\n\n# For Without-ToF data\ntrain_without_tof, val_without_tof = train_test_split(\n    without_tof_df,\n    test_size=0.2,\n    stratify=without_tof_df['gesture'],\n    random_state=42\n)\n\nprint(f\"Train with ToF: {len(train_with_tof)}, Val with ToF: {len(val_with_tof)}\")\nprint(f\"Train without ToF: {len(train_without_tof)}, Val without ToF: {len(val_without_tof)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:26.872618Z","iopub.execute_input":"2025-06-08T22:31:26.874970Z","iopub.status.idle":"2025-06-08T22:31:26.945666Z","shell.execute_reply.started":"2025-06-08T22:31:26.874907Z","shell.execute_reply":"2025-06-08T22:31:26.944350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"With ToF Gesture Distribution (Train):\")\nprint(train_with_tof['gesture'].value_counts(normalize=True))\n\nprint(\"\\nWithout ToF Gesture Distribution (Train):\")\nprint(train_without_tof['gesture'].value_counts(normalize=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:27.199846Z","iopub.execute_input":"2025-06-08T22:31:27.200336Z","iopub.status.idle":"2025-06-08T22:31:27.214401Z","shell.execute_reply.started":"2025-06-08T22:31:27.200313Z","shell.execute_reply":"2025-06-08T22:31:27.212938Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 4 : data prepration\n\n#### 4a : define useful feature groups","metadata":{}},{"cell_type":"code","source":"\nimu_base = ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']\nthermo_base = [f'thm_{i}' for i in range(1, 6)]\ntof_base = [f'tof_{i}_v{j}' for i in range(1, 6) for j in range(64)]\nstats = ['mean', 'std', 'min', 'max']\nimu_cols = [f\"{col}_{stat}\" for col in imu_base for stat in stats]\nthermo_cols = [f\"{col}_{stat}\" for col in thermo_base for stat in stats]\ntof_cols = [f\"{col}_{stat}\" for col in tof_base for stat in stats]\ndemo_cols = [\n    'age', 'sex', 'handedness', 'height_cm',\n    'shoulder_to_wrist_cm', 'elbow_to_wrist_cm', 'adult_child'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:27.841744Z","iopub.execute_input":"2025-06-08T22:31:27.843027Z","iopub.status.idle":"2025-06-08T22:31:27.851189Z","shell.execute_reply.started":"2025-06-08T22:31:27.842994Z","shell.execute_reply":"2025-06-08T22:31:27.849958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine all columns per model type\nfeatures_with_tof = imu_cols + thermo_cols + tof_cols + demo_cols\nfeatures_without_tof = imu_cols + thermo_cols + demo_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:28.671029Z","iopub.execute_input":"2025-06-08T22:31:28.671391Z","iopub.status.idle":"2025-06-08T22:31:28.677038Z","shell.execute_reply.started":"2025-06-08T22:31:28.671364Z","shell.execute_reply":"2025-06-08T22:31:28.675798Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4b : encode target levels","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\ntrain_with_tof['gesture_encoded'] = le.fit_transform(train_with_tof['gesture'])\nval_with_tof['gesture_encoded'] = le.transform(val_with_tof['gesture'])\n\ntrain_without_tof['gesture_encoded'] = le.transform(train_without_tof['gesture'])\nval_without_tof['gesture_encoded'] = le.transform(val_without_tof['gesture'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:29.236797Z","iopub.execute_input":"2025-06-08T22:31:29.237174Z","iopub.status.idle":"2025-06-08T22:31:29.253128Z","shell.execute_reply.started":"2025-06-08T22:31:29.237148Z","shell.execute_reply":"2025-06-08T22:31:29.251629Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4c : preprocessing to handle tof nans","metadata":{}},{"cell_type":"code","source":"def clean_tof(df, tof_columns):\n    for col in tof_columns:\n        if(df[col].isnull().any() or (df[col]==-1).any()):\n            mean = df[df[col]!=-1][col].mean()\n            df[col] = df[col].replace(-1,mean)\n    return df\n    \n# Already aggregated — no cleaning needed here\n# train_with_tof = clean_tof(train_with_tof, tof_cols)\n# val_with_tof = clean_tof(val_with_tof, tof_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:29.857568Z","iopub.execute_input":"2025-06-08T22:31:29.857907Z","iopub.status.idle":"2025-06-08T22:31:29.865697Z","shell.execute_reply.started":"2025-06-08T22:31:29.857885Z","shell.execute_reply":"2025-06-08T22:31:29.864563Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4d : features scaling","metadata":{}},{"cell_type":"code","source":"# ML models like LightGBM don't require scaling — but it's helpful if you later use models like SVM, MLP\n# We'll add it for completeness, especially for demographic values with different scales\nscaler_with_tof = StandardScaler()\nscaler_without_tof = StandardScaler()\n\nX_train_with_tof = scaler_with_tof.fit_transform(train_with_tof[features_with_tof])\nX_val_with_tof = scaler_with_tof.transform(val_with_tof[features_with_tof])\n\nX_train_without_tof = scaler_without_tof.fit_transform(train_without_tof[features_without_tof])\nX_val_without_tof = scaler_without_tof.transform(val_without_tof[features_without_tof])\n\nY_train_with_tof = train_with_tof['gesture_encoded']\nY_val_with_tof = val_with_tof['gesture_encoded']\n\nY_train_without_tof = train_without_tof['gesture_encoded']\nY_val_without_tof = val_without_tof['gesture_encoded']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:32.464232Z","iopub.execute_input":"2025-06-08T22:31:32.464592Z","iopub.status.idle":"2025-06-08T22:31:32.750399Z","shell.execute_reply.started":"2025-06-08T22:31:32.464563Z","shell.execute_reply":"2025-06-08T22:31:32.749561Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Step 5 : Modeling\n\n#### 5a : lightgbm models","metadata":{}},{"cell_type":"code","source":"model_with_tof = LGBMClassifier(\n    objective='multiclass',\n    num_class=len(le.classes_),\n    n_estimators=300,\n    learning_rate=0.05,\n    random_state=42\n)\n\nmodel_without_tof = LGBMClassifier(\n    objective='multiclass',\n    num_class=len(le.classes_),\n    n_estimators=300,\n    learning_rate=0.05,\n    random_state=42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:33.004378Z","iopub.execute_input":"2025-06-08T22:31:33.004705Z","iopub.status.idle":"2025-06-08T22:31:33.010778Z","shell.execute_reply.started":"2025-06-08T22:31:33.004686Z","shell.execute_reply":"2025-06-08T22:31:33.009715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Training model WITH ToF...\")\nmodel_with_tof.fit(X_train_with_tof, Y_train_with_tof)\n\nprint(\"Training model WITHOUT ToF...\")\nmodel_without_tof.fit(X_train_without_tof, Y_train_without_tof)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:31:33.275423Z","iopub.execute_input":"2025-06-08T22:31:33.275764Z","iopub.status.idle":"2025-06-08T22:38:52.787189Z","shell.execute_reply.started":"2025-06-08T22:31:33.275742Z","shell.execute_reply":"2025-06-08T22:38:52.786096Z"},"_kg_hide-output":true,"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 5b : evaluation","metadata":{}},{"cell_type":"code","source":"print(\"\\n🔍 Evaluation - Model WITH ToF:\")\npreds_with_tof = model_with_tof.predict(X_val_with_tof)\nprint(classification_report(Y_val_with_tof, preds_with_tof, target_names=le.classes_))\n\nprint(\"\\n🔍 Evaluation - Model WITHOUT ToF:\")\npreds_without_tof = model_without_tof.predict(X_val_without_tof)\nprint(classification_report(Y_val_without_tof, preds_without_tof, target_names=le.classes_))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:52.788744Z","iopub.execute_input":"2025-06-08T22:38:52.789066Z","iopub.status.idle":"2025-06-08T22:38:53.266353Z","shell.execute_reply.started":"2025-06-08T22:38:52.789035Z","shell.execute_reply":"2025-06-08T22:38:53.265115Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 6 : Post Prediction\n\n#### 6a : Analysis plot feature importance","metadata":{}},{"cell_type":"code","source":"# Create a directory to store saved models\nos.makedirs(\"saved_models/baseline/lgbm\", exist_ok=True)\n\n# Save the model WITH ToF\njoblib.dump(model_with_tof, \"saved_models/baseline/lgbm/model_with_tof.pkl\")\njoblib.dump(scaler_with_tof, \"saved_models/baseline/lgbm/scaler_with_tof.pkl\")\njoblib.dump(features_with_tof, \"saved_models/baseline/lgbm/features_with_tof_cols.pkl\")\n\n# Save the model WITHOUT ToF\njoblib.dump(model_without_tof, \"saved_models/baseline/lgbm/model_without_tof.pkl\")\njoblib.dump(scaler_without_tof, \"saved_models/baseline/lgbm/scaler_without_tof.pkl\")\njoblib.dump(features_without_tof, \"saved_models/baseline/lgbm/features_without_tof_cols.pkl\")\n\nprint(\"✅ All models and pipeline components saved to 'saved_models/' directory.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:53.267209Z","iopub.execute_input":"2025-06-08T22:38:53.267437Z","iopub.status.idle":"2025-06-08T22:38:54.019517Z","shell.execute_reply.started":"2025-06-08T22:38:53.267419Z","shell.execute_reply":"2025-06-08T22:38:54.018072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_feature_importance(model, feature_names, title, top_n=25):\n    importance = model.feature_importances_\n    indices = np.argsort(importance[-top_n:])\n    \n    plt.figure(figsize=(10,8))\n    plt.title(title)\n    plt.barh(range(top_n), importance[indices], align='center')\n    plt.yticks(range(top_n), [feature_names[i] for i in indices])\n    plt.xlabel(\"Importance score\")\n    plt.tight_layout()\n    plt.show()\n\n# Plot for model WITH ToF\nplot_feature_importance(model_with_tof, features_with_tof, \"Top Features - Model WITH ToF\")\n\n# Plot for model WITHOUT ToF\nplot_feature_importance(model_without_tof, features_without_tof, \"Top Features - Model WITHOUT ToF\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:54.021330Z","iopub.execute_input":"2025-06-08T22:38:54.021645Z","iopub.status.idle":"2025-06-08T22:38:54.868190Z","shell.execute_reply.started":"2025-06-08T22:38:54.021621Z","shell.execute_reply":"2025-06-08T22:38:54.867079Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 6b : investigate misclassified gestures","metadata":{}},{"cell_type":"code","source":"def plot_confusion(y_true, y_pred, labels, title):\n    cm = confusion_matrix(y_true, y_pred)\n    \n    plt.figure(figsize=(12,10))\n    sns.heatmap(cm, annot=True, fmt='d', xticklabels=labels, yticklabels=labels, cmap='Blues')\n    plt.title(title)\n    plt.xlabel('Predicted')\n    plt.ylabel('Actual')\n    plt.tight_layout()\n    plt.show()\n\n# Use this on your validation predictions\nplot_confusion(Y_val_with_tof, preds_with_tof, le.classes_, \"Confusion Matrix - With tof-\")\nplot_confusion(Y_val_without_tof, preds_without_tof, le.classes_, \"Confusion Matrix - Without tof-\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:42:18.148969Z","iopub.execute_input":"2025-06-08T22:42:18.149827Z","iopub.status.idle":"2025-06-08T22:42:20.513504Z","shell.execute_reply.started":"2025-06-08T22:42:18.149789Z","shell.execute_reply":"2025-06-08T22:42:20.512379Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"🔍 Confusion Matrix Insights (Key Takeaways)\n1. Some classes are perfectly or near-perfectly learned\n\n- ✅ “Feel around in tray and pull out an object” – 14/14 correct\n- ✅ “Text on phone” – 52/57 correct\n- ✅ “Drink from bottle/cup” – 13/14 correct\n- ✅ “Glasses on/off” – 12/13 correct\n\nImplication: These classes have distinctive features — good ToF signals, clear hand-object interaction, or consistent posture. We can examine what features dominate for these to extract prototype patterns.\n\n2. Some classes show wide confusion\n\n- ⚠️ “Eyebrow - pull hair”, “Eyelash - pull hair”, “Forehead - pull hairline”\n    - Strong confusion among each other, especially symmetrical areas (e.g., eye, eyebrow, forehead)\n- ⚠️ “Neck - pinch skin” vs “Neck - scratch”\n    - Nearly indistinguishable — tons of cross-label misclassifications\n- ⚠️ “Write name on leg” vs “Pull air toward your face”, “Wave hello”, “Write name in air”\n    - Confusion with gestures involving arm motion in mid-air\n\nImplication: For these, current features don't distinguish location or motion intent. This is our primary zone for feature engineering.\n\n3. Some classes are low recall or F1 simply due to confusion with many others\n\n- “Eyebrow pull hair” true positives = 18, but 30+ samples predicted elsewhere\n- “Write name in air” → many go to “Wave hello” and vice versa\n\nImplication: For these, feature overlap is likely. We’ll need to boost motion trajectory, hand location, or contextual timing.\n\nNext steps : We will first proceed with finetuning setup. If other models perform better we will have a better baseline, in case they don't and start bottlenecking around this we will know that feature engineering is needed from this point forward. In other words this will become our baseline.\n\n### 7 : finetuning setup","metadata":{}},{"cell_type":"code","source":"rf_params = {\n    'classifier__n_estimators': [100, 200, 300],\n    'classifier__max_depth': [10,20, None],\n    'classifier__min_samples_split': [2,5,10],\n    'classifier__min_samples_leaf': [1,2,4]\n}\n\n\"\"\"for quick execution of notebook I suggest you not to use Gradient Boosting, \nthe performance metrics of random forest and gradient boosting were similar.\"\"\"\ngb_params = {\n    'classifier__n_estimators': [100,200],\n    'classifier__learning_rate': [0.01, 0.1],\n    'classifier__max_depth': [3,5,7]\n}\n\nxgb_params = {\n    'classifier__n_estimators': [100,200],\n    'classifier__learning_rate': [0.01, 0.1],\n    'classifier__max_depth': [3,5,7],\n    'classifier__sub_sample': [0.8,1.0],\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:42:25.454158Z","iopub.execute_input":"2025-06-08T22:42:25.454485Z","iopub.status.idle":"2025-06-08T22:42:25.461795Z","shell.execute_reply.started":"2025-06-08T22:42:25.454460Z","shell.execute_reply":"2025-06-08T22:42:25.460393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tune_model(model, param_grid ,x_train, y_train, x_val, y_val, model_name):\n    pipe = Pipeline([\n        ('scaler', StandardScaler()),\n        ('imputer', SimpleImputer(strategy='mean')),\n        ('classifier', model)\n    ])\n\n    search = RandomizedSearchCV(pipe, param_grid, n_iter=10, cv=3, scoring='f1_weighted', n_jobs=-1, verbose=1)\n    search.fit(x_train, y_train)\n\n    best_model = search.best_estimator_\n    y_pred = best_model.predict(x_val)\n\n    print(f\"\\n📊 {model_name} Report:\\n\")\n    print(classification_report(y_val, y_pred))\n\n    os.makedirs('saved_models/optimized', exist_ok=True)\n    joblib.dump(best_model, f\"saved_models/optimized/{model_name}_model.joblib\")\n    return best_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:42:25.843645Z","iopub.execute_input":"2025-06-08T22:42:25.844109Z","iopub.status.idle":"2025-06-08T22:42:25.854348Z","shell.execute_reply.started":"2025-06-08T22:42:25.844080Z","shell.execute_reply":"2025-06-08T22:42:25.852940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rf_model = tune_model(RandomForestClassifier(random_state=42), rf_params, X_train_with_tof, Y_train_with_tof, X_val_with_tof, Y_val_with_tof, \"RandomForest\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:42:26.285167Z","iopub.execute_input":"2025-06-08T22:42:26.285646Z","execution_failed":"2025-06-08T22:42:50.693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gb_model = tune_model(GradientBoostingClassifier(random_state=42), gb_params, X_train_with_tof, Y_train_with_tof, X_val_with_tof, Y_val_with_tof, \"GradientBoosting\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-06-08T22:42:50.693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_model = tune_model(XGBClassifier(use_label_encoder=False, eval_metric='mlogloss', random_state=42), xgb_params, X_train_with_tof, Y_train_with_tof, X_val_with_tof, Y_val_with_tof, \"XGBoost\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:54.918979Z","iopub.status.idle":"2025-06-08T22:38:54.919329Z","shell.execute_reply.started":"2025-06-08T22:38:54.919136Z","shell.execute_reply":"2025-06-08T22:38:54.919153Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 7b : ensemble the best","metadata":{}},{"cell_type":"code","source":"rf_model = joblib.load(\"saved_models/optimized/RandomForest_model.joblib\")\ngb_model = joblib.load(\"saved_models/optimized/GradientBoosting_model.joblib\")\nxgb_model = joblib.load(\"saved_models/optimized/XGBoost_model.joblib\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:54.921942Z","iopub.status.idle":"2025-06-08T22:38:54.923002Z","shell.execute_reply.started":"2025-06-08T22:38:54.922817Z","shell.execute_reply":"2025-06-08T22:38:54.922835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use soft voting (uses predicted probabilities)\nvoting_clf = VotingClassifier(\n    estimators = [\n        ('rf', rf_model.named_steps['classifier']),\n        ('gb', gb_model.named_steps['classifier']),\n        ('xgb', xgb_model.named_steps['classifier']),\n    ],\n    voting = 'soft'\n)\n\nfinal_pipeline = Pipeline([\n    ('scaler', StandardScaler()),\n    ('imputer', SimpleImputer(strategy='mean')),\n    ('classifier', voting_clf)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:54.924475Z","iopub.status.idle":"2025-06-08T22:38:54.924842Z","shell.execute_reply.started":"2025-06-08T22:38:54.924664Z","shell.execute_reply":"2025-06-08T22:38:54.924679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_pipeline.fit(X_train_with_tof, Y_train_with_tof)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:54.925828Z","iopub.status.idle":"2025-06-08T22:38:54.926203Z","shell.execute_reply.started":"2025-06-08T22:38:54.926021Z","shell.execute_reply":"2025-06-08T22:38:54.926038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_ensemble = final_pipeline.predict(X_val_with_tof)\nprint(\"\\n🔮 Ensemble Model Report:\\n\")\nprint(classification_report(Y_val_with_tof, y_pred_ensemble))\n\njoblib.dump(final_pipeline, \"saved_models/optimized/Ensemble_model.joblib\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T22:38:54.927154Z","iopub.status.idle":"2025-06-08T22:38:54.927517Z","shell.execute_reply.started":"2025-06-08T22:38:54.927295Z","shell.execute_reply":"2025-06-08T22:38:54.927307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 8 : Analysis post model tuning and further steps\n\non analysis it is clear that lgbm (without optimisation) and xgb (with some parameter tuning) are plateuing around 0.54 f1 score. this might very possibly be happening due to removal of temporal features and their replacement by mean, min-max and std. This was a planned move as we wanted a baseline for the models we are further going to test.","metadata":{}}]}