{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ensemble Modeling for BFRB Sensor Data Classification\n\nThis notebook demonstrates an ensemble approach for classifying body-focused repetitive behaviors (BFRBs) using sensor data from the CMI competition. The following models are used in the ensemble:\n\n- **LightGBM**\n- **XGBoost**\n- **CatBoost**\n\n## Feature Selection\n\nFeatures are constructed as follows:\n- For each sequence, statistical features (mean, std, min, max, skew, kurtosis) are extracted from all numeric sensor columns (IMU, thermopile, ToF, etc.).\n- Demographic features from the `train_demographics.csv` file (such as age, sex, handedness, height, etc.) are merged and included as input features for the models.\n- The final feature set for each sequence includes all aggregated sensor statistics and all demographic columns except for the subject identifier.","metadata":{}},{"cell_type":"markdown","source":"for model submission checkout this notebook: https://www.kaggle.com/code/llkh0a/ensemble-inference/","metadata":{}},{"cell_type":"markdown","source":"# Data Loading & Exploration\nLoad the sensor and demographic data, and explore their structure.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# Read data\ntrain = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntrain_demo = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\n\nprint('Train shape:', train.shape)\nprint('Train demographics shape:', train_demo.shape)\nprint('Train columns:', train.columns.tolist())\nprint(train.head())","metadata":{"execution":{"iopub.execute_input":"2025-06-08T16:24:27.911234Z","iopub.status.busy":"2025-06-08T16:24:27.910978Z","iopub.status.idle":"2025-06-08T16:25:00.038366Z","shell.execute_reply":"2025-06-08T16:25:00.037739Z","shell.execute_reply.started":"2025-06-08T16:24:27.911191Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Cleaning\nCheck and handle missing values and outliers in sensor data.","metadata":{}},{"cell_type":"code","source":"# Check missing values\nmissing = train.isnull().sum()\nprint('Missing values per column:')\nprint(missing[missing > 0])\n\n# Replace -1 in ToF columns with NaN for easier statistics\ntof_cols = [col for col in train.columns if col.startswith('tof_')]\ntrain[tof_cols] = train[tof_cols].replace(-1, np.nan)","metadata":{"execution":{"iopub.execute_input":"2025-06-08T16:25:00.039774Z","iopub.status.busy":"2025-06-08T16:25:00.039587Z","iopub.status.idle":"2025-06-08T16:25:02.712799Z","shell.execute_reply":"2025-06-08T16:25:02.712256Z","shell.execute_reply.started":"2025-06-08T16:25:00.03976Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\nExtract statistical features for each sequence and merge with demographics.","metadata":{}},{"cell_type":"code","source":"def extract_features(df):\n    feats = []\n    # Only use numeric columns for aggregation\n    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()\n    # Remove columns that should not be aggregated\n    exclude_cols = ['row_id', 'sequence_id', 'sequence_counter', 'subject']\n    numeric_cols = [c for c in numeric_cols if c not in exclude_cols]\n    for seq_id, group in df.groupby('sequence_id'):\n        feat = {'sequence_id': seq_id}\n        for col in numeric_cols:\n            feat[col + '_mean'] = group[col].mean()\n            feat[col + '_std'] = group[col].std()\n            feat[col + '_min'] = group[col].min()\n            feat[col + '_max'] = group[col].max()\n            feat[col + '_skew'] = group[col].skew()\n            feat[col + '_kurt'] = group[col].kurt()\n            feat[col + '_missing'] = group[col].isnull().sum()\n        # New feature: total missing values in sequence\n        feat['total_missing'] = group.isnull().sum().sum()\n        # New feature: duration of sequence\n        feat['duration'] = group['sequence_counter'].max() - group['sequence_counter'].min()\n        # New feature: IMU signal energy\n        for axis in ['acc_x', 'acc_y', 'acc_z']:\n            if axis in group.columns:\n                feat[axis + '_energy'] = (group[axis] ** 2).sum()\n        feat['subject'] = group['subject'].iloc[0]\n        feats.append(feat)\n    return pd.DataFrame(feats)\n\nX = extract_features(train)\n# Merge demographic features (excluding subject key)\ndemographic_features = [col for col in train_demo.columns if col != 'subject']\nX = X.merge(train_demo, on='subject', how='left')\n# Fill any remaining missing values in features with column mean (or 0 as fallback)\nX = X.fillna(X.mean(numeric_only=True)).fillna(0)\n# Add demographic features to training set\nfeature_cols = [col for col in X.columns if col not in ['sequence_id', 'subject']]","metadata":{"execution":{"iopub.execute_input":"2025-06-08T16:25:02.715735Z","iopub.status.busy":"2025-06-08T16:25:02.715578Z","iopub.status.idle":"2025-06-08T16:29:28.793327Z","shell.execute_reply":"2025-06-08T16:29:28.792759Z","shell.execute_reply.started":"2025-06-08T16:25:02.715722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target Mapping\nMap all gestures not in the target list to 'non_target' for proper evaluation.","metadata":{}},{"cell_type":"code","source":"target_gestures = [\n    'Above ear - pull hair',\n    'Cheek - pinch skin',\n    'Eyebrow - pull hair',\n    'Eyelash - pull hair',\n    'Forehead - pull hairline',\n    'Forehead - scratch',\n    'Neck - pinch skin',\n    'Neck - scratch',\n]\nnon_target_gestures = [\n    'Write name on leg',\n    'Wave hello',\n    'Glasses on/off',\n    'Text on phone',\n    'Write name in air',\n    'Feel around in tray and pull out an object',\n    'Scratch knee/leg skin',\n    'Pull air toward your face',\n    'Drink from bottle/cup',\n    'Pinch knee/leg skin'\n]\n\n# Map all gestures not in target_gestures to 'non_target'\ntrain['gesture_mapped'] = train['gesture'].apply(lambda x: x if x in target_gestures else 'non_target')\ny = train.groupby('sequence_id')['gesture_mapped'].first().values","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\nfrom sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\ny_enc = le.fit_transform(y)\njoblib.dump(le, 'label_encoder.joblib')\n# Print label map for reference\nlabel_map = {i: label for i, label in enumerate(le.classes_)}\nprint('Label map:', label_map)","metadata":{"execution":{"iopub.execute_input":"2025-06-08T16:30:05.204311Z","iopub.status.busy":"2025-06-08T16:30:05.204002Z","iopub.status.idle":"2025-06-08T16:30:05.212742Z","shell.execute_reply":"2025-06-08T16:30:05.212158Z","shell.execute_reply.started":"2025-06-08T16:30:05.204289Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training, Hyperparameter Tuning, and Ensemble\n\nThis section covers hyperparameter tuning for base models, followed by ensemble training and evaluation using the competition metric.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV, train_test_split\nfrom sklearn.metrics import make_scorer, accuracy_score, f1_score\nfrom lightgbm import LGBMClassifier\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.ensemble import VotingClassifier\nimport joblib\nimport numpy as np\n\n# Helper for binary F1\ndef to_binary(y, target_gestures, label_encoder):\n    y_labels = label_encoder.inverse_transform(y)\n    return [1 if gesture in target_gestures else 0 for gesture in y_labels]\n\n# Custom scorer for competition metric\ndef competition_metric(y_true, y_pred):\n    binary_f1 = f1_score(to_binary(y_true, target_gestures, le), to_binary(y_pred, target_gestures, le))\n    macro_f1 = f1_score(y_true, y_pred, average='macro')\n    return (binary_f1 + macro_f1) / 2\n\ncompetition_scorer = make_scorer(competition_metric, greater_is_better=True)\n\n# Split data\nX_train, X_val, y_train, y_val = train_test_split(\n    X[feature_cols], y_enc, test_size=0.2, random_state=42, stratify=y_enc)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Hyperparameter tuning for LightGBM\nlgbm_params = {\n    'n_estimators': [100, 200, 300],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'num_leaves': [15, 31, 63],\n    'max_depth': [3, 5, 7, -1],\n}\nlgbm_search = RandomizedSearchCV(\n    LGBMClassifier(),\n    param_distributions=lgbm_params,\n    n_iter=10,\n    scoring=competition_scorer,\n    cv=3,\n    verbose=2,\n    random_state=42,\n    n_jobs=-1\n)\nlgbm_search.fit(X_train, y_train)\nprint('Best LightGBM params:', lgbm_search.best_params_)\nprint('Best LightGBM score:', lgbm_search.best_score_)\nlgbm_best = lgbm_search.best_estimator_\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Hyperparameter tuning for XGBoost\nxgb_params = {\n    'n_estimators': [100, 200, 300],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'max_depth': [3, 5, 7],\n    'subsample': [0.7, 0.85, 1.0],\n    'colsample_bytree': [0.7, 0.85, 1.0],\n}\nxgb_search = RandomizedSearchCV(\n    XGBClassifier(use_label_encoder=False, eval_metric='mlogloss'),\n    param_distributions=xgb_params,\n    n_iter=10,\n    scoring=competition_scorer,\n    cv=3,\n    verbose=2,\n    random_state=42,\n    n_jobs=-1\n)\nxgb_search.fit(X_train, y_train)\nprint('Best XGBoost params:', xgb_search.best_params_)\nprint('Best XGBoost score:', xgb_search.best_score_)\nxgb_best = xgb_search.best_estimator_\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Hyperparameter tuning for CatBoost\ncat_params = {\n    'iterations': [100, 200, 300],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'depth': [3, 5, 7],\n    'l2_leaf_reg': [1, 3, 5, 7],\n}\ncat_search = RandomizedSearchCV(\n    CatBoostClassifier(verbose=0),\n    param_distributions=cat_params,\n    n_iter=10,\n    scoring=competition_scorer,\n    cv=3,\n    verbose=2,\n    random_state=42,\n    n_jobs=-1\n)\ncat_search.fit(X_train, y_train)\nprint('Best CatBoost params:', cat_search.best_params_)\nprint('Best CatBoost score:', cat_search.best_score_)\ncat_best = cat_search.best_estimator_","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Ensemble with best estimators\nensemble = VotingClassifier(estimators=[\n    ('lgbm', lgbm_best),\n    ('xgb', xgb_best),\n    ('cat', cat_best)\n], voting='soft')\n\nensemble.fit(X_train, y_train)\ny_pred = ensemble.predict(X_val)\nprint('Validation accuracy:', accuracy_score(y_val, y_pred))\n\n# Save the ensemble model to a file\njoblib.dump(ensemble, 'ensemble_model.joblib')\nprint('Model saved as ensemble_model.joblib')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score\n# Calculate competition metrics\ny_true = y_val\nbinary_f1 = f1_score(to_binary(y_true, target_gestures, le), to_binary(y_pred, target_gestures, le))\nmacro_f1 = f1_score(y_true, y_pred, average='macro')\nprint('Binary F1 score:', binary_f1)\nprint('Macro F1 score:', macro_f1)\ncompetition_metric_score = competition_metric(y_val, y_pred)\n\nprint('Competition metric score on validation set:', competition_metric_score)","metadata":{},"outputs":[],"execution_count":null}]}