{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12542187,"sourceType":"datasetVersion","datasetId":7918298}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
🚆 Model Training
\n\nFor Competition notebook visit ---> [https://www.kaggle.com/code/taimour/cmi-deep-dive-eda-boosters](https://www.kaggle.com/code/taimour/cmi-deep-dive-eda-boosters)\n\nDue to time limitations, model was trained separately in this notebook.","metadata":{}},{"cell_type":"markdown","source":"#
📚 Libraries / Packages
","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nimport joblib\n\nimport catboost\nfrom catboost import CatBoostClassifier\nimport lightgbm as lgb\nimport xgboost as xgb\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import LabelEncoder, RobustScaler\n# from sklearn.impute import KNNImputer\n# from sklearn.decomposition import PCA\n# from sklearn.pipeline import Pipeline\n# from sklearn.ensemble import VotingClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:14.112116Z","iopub.execute_input":"2025-08-08T10:15:14.112338Z","iopub.status.idle":"2025-08-08T10:15:23.834381Z","shell.execute_reply.started":"2025-08-08T10:15:14.112315Z","shell.execute_reply":"2025-08-08T10:15:23.833721Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
📖 Load Data
","metadata":{}},{"cell_type":"code","source":"cleaned_data = pl.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data-cleaned/cmi_bfrb_cleaned_data_full.csv')\n# train = pl.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\n# train_demo = pl.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\n\n# # Drop these columns from training data\n# train = train.drop(['phase', 'orientation', 'behavior', 'sequence_type'])\n\n# train = train.join(train_demo,on=\"subject\",how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:23.834996Z","iopub.execute_input":"2025-08-08T10:15:23.835540Z","iopub.status.idle":"2025-08-08T10:15:31.939390Z","shell.execute_reply.started":"2025-08-08T10:15:23.835521Z","shell.execute_reply":"2025-08-08T10:15:31.938693Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
💎 Feature Engineering
\n\n**Helper Functions**","metadata":{}},{"cell_type":"code","source":"def remove_gravity_from_acc(acc_data, rot_data):\n\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = acc_data\n\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = rot_data\n\n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n \n gravity_world = np.array([0, 0, 9.81])\n\n for i in range(num_samples):\n if np.all(np.isnan(quat_values[i])) or np.all(np.isclose(quat_values[i], 0)):\n linear_accel[i, :] = acc_values[i, :] \n continue\n\n try:\n rotation = R.from_quat(quat_values[i])\n gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n except ValueError:\n linear_accel[i, :] = acc_values[i, :]\n \n return linear_accel\n\ndef calculate_angular_velocity_from_quat(rot_data, time_delta=1/200): # Assuming 200Hz sampling rate\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = rot_data\n\n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n\n for i in range(num_samples - 1):\n q_t = quat_values[i]\n q_t_plus_dt = quat_values[i+1]\n\n if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n continue\n\n try:\n rot_t = R.from_quat(q_t)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n\n # Calculate the relative rotation\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n # Convert delta rotation to angular velocity vector\n # The rotation vector (Euler axis * angle) scaled by 1/dt\n # is a good approximation for small delta_rot\n angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n except ValueError:\n # If quaternion is invalid, angular velocity remains zero\n pass\n \n return angular_vel\n\ndef calculate_angular_distance(rot_data):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = rot_data\n\n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n\n for i in range(num_samples - 1):\n q1 = quat_values[i]\n q2 = quat_values[i+1]\n\n if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n angular_dist[i] = 0 # Или np.nan, в зависимости от желаемого поведения\n continue\n try:\n # Converting quaternions to Rotation objects\n r1 = R.from_quat(q1)\n r2 = R.from_quat(q2)\n\n # Calculating the angular distance: 2 * arccos(|real(p * q*)|)\n # where q* is the conjugate of quaternion q\n # In scipy.spatial.transform.Rotation, r1.inv() * r2 gives the relative rotation.\n # The angle of this relative rotation is the angular distance.\n relative_rotation = r1.inv() * r2\n \n # The angle of the rotation vector corresponds to the angular distance\n # The norm of the rotation vector is the angle in radians\n angle = np.linalg.norm(relative_rotation.as_rotvec())\n angular_dist[i] = angle\n except ValueError:\n angular_dist[i] = 0 # In case of invalid quaternions\n pass\n \n return angular_dist","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:31.941037Z","iopub.execute_input":"2025-08-08T10:15:31.941529Z","iopub.status.idle":"2025-08-08T10:15:31.953234Z","shell.execute_reply.started":"2025-08-08T10:15:31.941509Z","shell.execute_reply":"2025-08-08T10:15:31.952479Z"},"_kg_hide-input":false},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**IMU**","metadata":{}},{"cell_type":"code","source":"def feature_engineering_imu(data:pl.DataFrame):\n data = data.to_pandas()\n data['acc_mag'] = np.sqrt(data['acc_x']**2 + data['acc_y']**2 + data['acc_z']**2)\n data['rot_angle'] = 2 * np.arccos(data['rot_w'].clip(-1, 1))\n data['acc_mag_jerk'] = data.groupby('sequence_id')['acc_mag'].diff().fillna(0)\n data['rot_angle_vel'] = data.groupby('sequence_id')['rot_angle'].diff().fillna(0)\n\n linear_accel_list = []\n for _, group in data.groupby('sequence_id'):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']]\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['linear_acc_x', 'linear_acc_y', 'linear_acc_z'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list)\n data = pd.concat([data, df_linear_accel], axis=1)\n data['linear_acc_mag'] = np.sqrt(data['linear_acc_x']**2 + data['linear_acc_y']**2 + data['linear_acc_z']**2)\n data['linear_acc_mag_jerk'] = data.groupby('sequence_id')['linear_acc_mag'].diff().fillna(0)\n angular_vel_list = []\n for _, group in data.groupby('sequence_id'):\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['angular_vel_x', 'angular_vel_y', 'angular_vel_z'], index=group.index))\n \n df_angular_vel = pd.concat(angular_vel_list)\n data = pd.concat([data, df_angular_vel], axis=1)\n \n print(\" Calculating angular distance between successive quaternions...\")\n angular_distance_list = []\n for _, group in data.groupby('sequence_id'):\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['angular_distance'], index=group.index))\n \n df_angular_distance = pd.concat(angular_distance_list)\n data = pd.concat([data, df_angular_distance], axis=1)\n data = pl.from_pandas(data)\n return data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:31.954060Z","iopub.execute_input":"2025-08-08T10:15:31.954328Z","iopub.status.idle":"2025-08-08T10:15:31.987082Z","shell.execute_reply.started":"2025-08-08T10:15:31.954305Z","shell.execute_reply":"2025-08-08T10:15:31.986518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = feature_engineering_imu(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:31.987805Z","iopub.execute_input":"2025-08-08T10:15:31.988248Z","iopub.status.idle":"2025-08-08T10:15:32.004363Z","shell.execute_reply.started":"2025-08-08T10:15:31.988225Z","shell.execute_reply":"2025-08-08T10:15:32.003744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:32.004927Z","iopub.execute_input":"2025-08-08T10:15:32.005099Z","iopub.status.idle":"2025-08-08T10:15:32.020398Z","shell.execute_reply.started":"2025-08-08T10:15:32.005085Z","shell.execute_reply":"2025-08-08T10:15:32.019717Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Statistical**","metadata":{}},{"cell_type":"code","source":"def feature_engineering_stat(data:pl.DataFrame):\n non_sensor_cols = []\n if \"gesture\" in data.columns:\n non_sensor_cols = [\"gesture\"]\n \n # All numeric sensor columns (everything except id, demo, target)\n stat_cols = [\n c for c in data.columns\n if c not in non_sensor_cols + [\"sequence_id\", \"row_id\",\"sequence_counter\",\"subject\"]\n ]\n \n # Build aggregation expressions\n agg_exprs = []\n \n # full-stats bundle for sensor columns\n for c in stat_cols:\n agg_exprs += [\n pl.col(c).mean().alias(f\"{c}_mean\"),\n pl.col(c).std().alias(f\"{c}_std\"),\n pl.col(c).var().alias(f\"{c}_var\"),\n pl.col(c).quantile(0.25).alias(f\"{c}_q25\"),\n pl.col(c).median().alias(f\"{c}_q50\"),\n pl.col(c).quantile(0.75).alias(f\"{c}_q75\"),\n pl.col(c).max().alias(f\"{c}_max\"),\n pl.col(c).min().alias(f\"{c}_min\"),\n pl.col(c).first().alias(f\"{c}_first\"),\n pl.col(c).last().alias(f\"{c}_last\"),\n pl.col(c).quantile(0.25, \"nearest\").alias(f\"{c}_t25\"),\n pl.col(c).quantile(0.75, \"nearest\").alias(f\"{c}_t75\"),\n (pl.col(c).last() - pl.col(c).first()).alias(f\"{c}_delta\"),\n pl.corr(\"sequence_counter\", c).alias(f\"{c}_corr_time\"),\n pl.col(c).diff().mean().alias(f\"{c}_diff_mean\"),\n pl.col(c).diff().std().alias(f\"{c}_diff_std\"),\n pl.col(c).skew().alias(f\"{c}_skew\"),\n pl.col(c).kurtosis().alias(f\"{c}_kurt\"),\n pl.col(c).diff().abs().gt(0).sum().alias(f\"{c}_n_changes\")\n ]\n agg_exprs += [\n pl.when(pl.col(\"sequence_counter\") < 0.1 * pl.max(\"sequence_counter\"))\n .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg1_mean\"),\n pl.when(pl.col(\"sequence_counter\") > 0.9 * pl.max(\"sequence_counter\"))\n .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg3_mean\"),\n ]\n \n # first() for demographics and target\n agg_exprs += [\n pl.col(c).first().alias(c) for c in non_sensor_cols\n ]\n \n # Group-by and aggregate\n cleaned_data = (\n data\n .group_by(\"sequence_id\", maintain_order=True)\n .agg(agg_exprs)\n )\n return cleaned_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:32.020996Z","iopub.execute_input":"2025-08-08T10:15:32.021218Z","iopub.status.idle":"2025-08-08T10:15:32.034357Z","shell.execute_reply.started":"2025-08-08T10:15:32.021200Z","shell.execute_reply":"2025-08-08T10:15:32.033705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_demographic_target_cols = [\n# \"gesture\"\n# ]\n# cleaned_data = feature_engineering_stat(train)\n# cleaned_data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:32.036488Z","iopub.execute_input":"2025-08-08T10:15:32.036845Z","iopub.status.idle":"2025-08-08T10:15:32.050086Z","shell.execute_reply.started":"2025-08-08T10:15:32.036825Z","shell.execute_reply":"2025-08-08T10:15:32.049558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cleaned_data.write_csv('cmi_bfrb_cleaned_data_full.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:32.050789Z","iopub.execute_input":"2025-08-08T10:15:32.051000Z","iopub.status.idle":"2025-08-08T10:15:32.062871Z","shell.execute_reply.started":"2025-08-08T10:15:32.050977Z","shell.execute_reply":"2025-08-08T10:15:32.062075Z"},"_kg_hide-input":false},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
🎯 Scaling & Encoding
","metadata":{}},{"cell_type":"code","source":"# Assume cleaned_data is already a Polars DataFrame\ntarget_col = \"gesture\"\n\n# --- Convert Polars DataFrame to Pandas only if needed ---\n# CatBoost does not yet fully support Polars directly\ndf = cleaned_data.to_pandas()\n\n# --- Define X and y properly ---\nX = df.drop(columns=[target_col, \"sequence_id\"]) # Feature matrix\ny = df[target_col].values # Target\n# Estimate number of classes from your target variable `y`\nnum_classes = len(set(y)) # or len(np.unique(y))\n\n# Encode target\nle = LabelEncoder()\ny = le.fit_transform(y)\n\njoblib.dump(le, 'Target_LabelEncoder.joblib')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:32.063544Z","iopub.execute_input":"2025-08-08T10:15:32.063751Z","iopub.status.idle":"2025-08-08T10:15:32.894195Z","shell.execute_reply.started":"2025-08-08T10:15:32.063735Z","shell.execute_reply":"2025-08-08T10:15:32.893442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Initialize the scaler\nrobustscaler = RobustScaler()\n# Fit on the integer columns of training data (changed to numpy array)\nX_scaled = robustscaler.fit_transform(X)\n#numpy array convert to panda's dataframe\nX = pd.DataFrame(X_scaled, columns=X.columns)\n\njoblib.dump(robustscaler, 'robustscaler.joblib')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:32.894994Z","iopub.execute_input":"2025-08-08T10:15:32.895215Z","iopub.status.idle":"2025-08-08T10:15:37.061648Z","shell.execute_reply.started":"2025-08-08T10:15:32.895192Z","shell.execute_reply":"2025-08-08T10:15:37.060908Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
🎲 Parameters for Models
","metadata":{}},{"cell_type":"code","source":"params_cat = {\n 'loss_function': 'MultiClass',\n 'learning_rate': 0.06,\n 'iterations': 3000,\n 'task_type': 'GPU',\n 'verbose' : False\n} \n\nparams_lgb = {\n 'objective': 'multiclass',\n 'num_class': num_classes,\n 'learning_rate': 0.1,\n 'n_estimators': 1000,\n 'max_bin' : 20,\n 'num_threads' : -1,\n 'device': 'gpu', # Requires LightGBM with GPU support compiled\n 'verbose': -1 # Silence output; set to 1 to see progress\n}\n\nparams_xgb = {\n 'objective': 'multi:softprob', # Equivalent to multiclass in LightGBM; outputs probabilities\n 'num_class': num_classes, # Required for multi-class in XGBoost\n 'learning_rate': 0.08, # Same as LightGBM's learning_rate\n 'n_estimators': 1200, # Same number of boosting rounds\n 'max_bin': 256, # XGBoost default is higher; LightGBM uses 20, but XGBoost typically uses 256\n 'n_jobs': -1, # Equivalent to num_threads in LightGBM\n 'tree_method': 'hist', # Enables GPU; requires XGBoost with GPU support\n 'device': 'cuda',\n 'verbosity': 0, # 0 = silent, 1 = info (similar to verbose=-1 in LightGBM)\n 'eval_metric': 'mlogloss' # Optional: for monitoring; multiclass logloss\n}","metadata":{"trusted":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2025-08-08T10:15:37.062494Z","iopub.execute_input":"2025-08-08T10:15:37.062762Z","iopub.status.idle":"2025-08-08T10:15:37.068130Z","shell.execute_reply.started":"2025-08-08T10:15:37.062745Z","shell.execute_reply":"2025-08-08T10:15:37.067390Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
↪️ Stratified K Fold
","metadata":{}},{"cell_type":"code","source":"# Initialize K-Fold\nn_splits = 5\nkfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n# During cross-validation, collect best iterations\ncat_best_iters = []\nlgb_best_iters = []\nxgb_best_iters = []\n\n# Store OOF predictions (for training meta-model)\noof_predictions = np.zeros((len(X), num_classes * 3)) # Final OOF from base models\noof_targets = np.empty(len(y), dtype=int) # True labels\n\n# Meta-feature generation: Collect predictions from each model\nprint(\"Starting Stacking with Early Stopping...\\n\")\n\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(X, y), start=1):\n print(f\"Training Fold {fold}/{n_splits}...\")\n\n X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n y_train, y_val = y[train_idx], y[val_idx]\n\n # Convert to appropriate formats if needed\n if isinstance(X_train, pd.DataFrame):\n pass # Already in correct format\n\n # --- 1. CatBoost Classifier ---\n model_cat = CatBoostClassifier(**params_cat)\n model_cat.fit(\n X_train, y_train,\n eval_set=(X_val, y_val),\n early_stopping_rounds=100,\n verbose=False\n )\n pred_cat = model_cat.predict_proba(X_val) # (n_samples, n_classes)\n\n # --- 2. LightGBM Classifier ---\n model_lgb = lgb.LGBMClassifier(**params_lgb)\n model_lgb.fit(\n X_train, y_train,\n eval_set=[(X_val, y_val)],\n eval_names=['valid'],\n callbacks=[lgb.early_stopping(stopping_rounds=100, verbose=True)]\n )\n pred_lgb = model_lgb.predict_proba(X_val)\n\n # --- 3. XGBoost Classifier ---\n model_xgb = xgb.XGBClassifier(**params_xgb)\n model_xgb.set_params(early_stopping_rounds=100) # Set before fit()\n model_xgb.fit(\n X_train, y_train,\n eval_set=[(X_val, y_val)],\n verbose=False\n )\n pred_xgb = model_xgb.predict_proba(X_val)\n\n # Average predictions (soft voting style) or concatenate as features\n # For stacking, we concatenate as features: shape = (n_samples, 3 * n_classes)\n stacked_features = np.hstack([pred_cat, pred_lgb, pred_xgb]) # (n, 3*C)\n\n # Store OOF predictions\n oof_predictions[val_idx] = stacked_features\n oof_targets[val_idx] = y_val\n\n # Record best iterations\n cat_best_iters.append(model_cat.get_best_iteration())\n lgb_best_iters.append(model_lgb.best_iteration_)\n xgb_best_iters.append(model_xgb.best_iteration)\n\n# Calculate average best iterations (with safety margin)\ncat_optimal = int(np.mean(cat_best_iters) * 1.15)\nlgb_optimal = int(np.mean(lgb_best_iters) * 1.15)\nxgb_optimal = int(np.mean(xgb_best_iters) * 1.15)\n\n# Print with detailed explanation\nprint(\"\\n📊 OPTIMAL ITERATIONS ANALYSIS\")\nprint(\"================================\")\nprint(f\"CatBoost: {np.mean(cat_best_iters):.1f} (avg from CV) → {cat_optimal} (with buffer)\")\nprint(f\"LightGBM: {np.mean(lgb_best_iters):.1f} (avg from CV) → {lgb_optimal} (with buffer)\")\nprint(f\"XGBoost: {np.mean(xgb_best_iters):.1f} (avg from CV) → {xgb_optimal} (with buffer)\")\nprint(\"================================\")\nprint(\"💡 Why buffer?\")\nprint(\"- Prevents underfitting when training on full dataset\")\nprint(\"- Accounts for slightly different data distribution\")\nprint(\"- Ensures we don't stop just before optimal performance\")\nprint(\"- Still much more efficient than training to max iterations\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:15:37.068867Z","iopub.execute_input":"2025-08-08T10:15:37.069047Z","iopub.status.idle":"2025-08-08T10:16:36.782154Z","shell.execute_reply.started":"2025-08-08T10:15:37.069032Z","shell.execute_reply":"2025-08-08T10:16:36.781457Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
🗳️ Meta Model
","metadata":{}},{"cell_type":"code","source":"# Train meta-model (level-2 model) on OOF predictions\nprint(\"\\nTraining meta-model (Logistic Regression)...\")\nmeta_model = LogisticRegression(\n multi_class='multinomial',\n max_iter=1000,\n n_jobs=-1,\n random_state=42\n)\nmeta_model.fit(oof_predictions, oof_targets)\n\n# Evaluate on OOF (out-of-fold) predictions\nfinal_oof_pred = meta_model.predict(oof_predictions)\nfinal_oof_proba = meta_model.predict_proba(oof_predictions)\n\nacc_oof = accuracy_score(oof_targets, final_oof_pred)\nf1_oof = f1_score(oof_targets, final_oof_pred, average='macro')\n\nprint(\"\\n====== Final OOF Results (after stacking) ======\")\nprint(f\"Accuracy: {acc_oof:.4f}\")\nprint(f\"Macro-F1 : {f1_oof:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:16:36.782899Z","iopub.execute_input":"2025-08-08T10:16:36.783150Z","iopub.status.idle":"2025-08-08T10:16:38.467206Z","shell.execute_reply.started":"2025-08-08T10:16:36.783131Z","shell.execute_reply":"2025-08-08T10:16:38.466459Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
🚅 Train the Model
","metadata":{}},{"cell_type":"code","source":"params_cat_optimal = {\n 'loss_function': 'MultiClass',\n 'learning_rate': 0.06,\n 'iterations': cat_optimal,\n 'task_type': 'GPU',\n 'verbose' : False\n} \n\nparams_lgb_optimal = {\n 'objective': 'multiclass',\n 'num_class': num_classes,\n 'learning_rate': 0.1,\n 'n_estimators': lgb_optimal,\n 'max_bin' : 20,\n 'num_threads' : -1,\n 'device': 'gpu', # Requires LightGBM with GPU support compiled\n 'verbose': -1 # Silence output; set to 1 to see progress\n}\n\nparams_xgb_optimal = {\n 'objective': 'multi:softprob', # Equivalent to multiclass in LightGBM; outputs probabilities\n 'num_class': num_classes, # Required for multi-class in XGBoost\n 'learning_rate': 0.08, # Same as LightGBM's learning_rate\n 'n_estimators': xgb_optimal, # Same number of boosting rounds\n 'max_bin': 256, # XGBoost default is higher; LightGBM uses 20, but XGBoost typically uses 256\n 'n_jobs': -1, # Equivalent to num_threads in LightGBM\n 'tree_method': 'hist', # Enables GPU; requires XGBoost with GPU support\n 'device': 'cuda',\n 'verbosity': 0, # 0 = silent, 1 = info (similar to verbose=-1 in LightGBM)\n 'eval_metric': 'mlogloss' # Optional: for monitoring; multiclass logloss\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:16:38.468025Z","iopub.execute_input":"2025-08-08T10:16:38.468301Z","iopub.status.idle":"2025-08-08T10:16:38.473300Z","shell.execute_reply.started":"2025-08-08T10:16:38.468258Z","shell.execute_reply":"2025-08-08T10:16:38.472584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def refit_all_models(X_train_full, y_train_full):\n \"\"\"\n Refit one instance of each model on the full dataset.\n \"\"\"\n print(\"Refitting one model per algorithm on full dataset...\")\n\n # CatBoost\n model_cat = CatBoostClassifier(**params_cat_optimal)\n model_cat.fit(X_train_full, y_train_full, verbose=False)\n\n # LightGBM\n model_lgb = lgb.LGBMClassifier(**params_lgb_optimal)\n model_lgb.fit(X_train_full, y_train_full)\n \n # XGBoost\n model_xgb = xgb.XGBClassifier(**params_xgb_optimal)\n model_xgb.fit(X_train_full, y_train_full, verbose=0)\n\n # Wrap in lists for consistent interface\n return model_cat, model_lgb, model_xgb\n\n# Refit All Models\nmodel_cat, model_lgb, model_xgb = refit_all_models(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:16:38.474156Z","iopub.execute_input":"2025-08-08T10:16:38.474641Z","iopub.status.idle":"2025-08-08T10:16:59.298197Z","shell.execute_reply.started":"2025-08-08T10:16:38.474618Z","shell.execute_reply":"2025-08-08T10:16:59.297643Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
💾 Save the Model
","metadata":{}},{"cell_type":"code","source":"joblib.dump(model_cat, f'model_cat.joblib')\njoblib.dump(model_lgb, f'model_lgb.joblib')\njoblib.dump(model_xgb, f'model_xgb.joblib')\njoblib.dump(meta_model, f'meta_model.joblib')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:16:59.299914Z","iopub.execute_input":"2025-08-08T10:16:59.300138Z","iopub.status.idle":"2025-08-08T10:16:59.395173Z","shell.execute_reply.started":"2025-08-08T10:16:59.300121Z","shell.execute_reply":"2025-08-08T10:16:59.394680Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
🌟 Predict
\n\nNot needed as we are just trainig the model. It is only kept here to be used for testing only, whenever needed.","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"# def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n# # data =sequence\n# data = sequence.join(demographics,on=\"subject\",how=\"left\")\n# # print(data.schema)\n# data = feature_engineering_imu(data)\n# cleaned_data = feature_engineering_stat(data)\n\n# pdf = cleaned_data.to_pandas().drop(columns=[\"sequence_id\"])\n\n# pdf_scaled = robustscaler.transform(pdf) # Shape: (1, n_features)\n \n# # Get predictions from base models (probabilities)\n# p_cat = model_cat.predict_proba(pdf_scaled) # (1, num_classes)\n# p_lgb = model_lgb.predict_proba(pdf_scaled)\n# p_xgb = model_xgb.predict_proba(pdf_scaled)\n \n# # Stack probabilities for meta-model\n# meta_features = np.hstack([p_cat, p_lgb, p_xgb]) # (1, 3*num_classes)\n \n# # Final prediction using meta-model\n# y_pred_encoded = meta_model.predict(meta_features) # Array of length 1\n# class_label = le.inverse_transform(y_pred_encoded)[0]\n# return class_label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:16:59.395600Z","iopub.execute_input":"2025-08-08T10:16:59.395771Z","iopub.status.idle":"2025-08-08T10:16:59.403366Z","shell.execute_reply.started":"2025-08-08T10:16:59.395756Z","shell.execute_reply":"2025-08-08T10:16:59.402752Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
🪐 Inference Server
\nNot needed as we are just trainig the model. It is only kept here to be used for testing only, whenever needed.","metadata":{}},{"cell_type":"code","source":"# inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\n# if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n# inference_server.serve()\n# else:\n# inference_server.run_local_gateway(\n# data_paths=(\n# '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n# '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n# )\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T10:16:59.404015Z","iopub.execute_input":"2025-08-08T10:16:59.404234Z","iopub.status.idle":"2025-08-08T10:17:01.379723Z","shell.execute_reply.started":"2025-08-08T10:16:59.404208Z","shell.execute_reply":"2025-08-08T10:17:01.378986Z"}},"outputs":[],"execution_count":null}]}