{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":242561727,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is the training notebook, if you want the prediction notebook...\n\nMy inference notebook is at [CMI 2025 - Inference](https://www.kaggle.com/code/coreymichaud/cmi-2025-inference)","metadata":{}},{"cell_type":"code","source":"# Libraries\nimport os\n\nimport pandas as pd\nimport numpy as np\nimport polars as pl\n\nimport joblib\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom metric import score, CompetitionMetric  # metric given by the competition\n\nimport xgboost as xgb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:05.369037Z","iopub.execute_input":"2025-06-03T18:33:05.369408Z","iopub.status.idle":"2025-06-03T18:33:07.204665Z","shell.execute_reply.started":"2025-06-03T18:33:05.369382Z","shell.execute_reply":"2025-06-03T18:33:07.203917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CONFIG\nCROSSVAL = 1\nTRAINING = 0\nSAVING = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:07.206059Z","iopub.execute_input":"2025-06-03T18:33:07.206484Z","iopub.status.idle":"2025-06-03T18:33:07.210353Z","shell.execute_reply.started":"2025-06-03T18:33:07.206458Z","shell.execute_reply":"2025-06-03T18:33:07.209535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Importing training data\ntrain = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\ntrain_demographics = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:07.211166Z","iopub.execute_input":"2025-06-03T18:33:07.211433Z","iopub.status.idle":"2025-06-03T18:33:40.153544Z","shell.execute_reply.started":"2025-06-03T18:33:07.211412Z","shell.execute_reply":"2025-06-03T18:33:40.152984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combining train with train_demographics ... should have 348 columns if successful\ntrain_merged = pd.merge(train, train_demographics, on = \"subject\", how = \"left\")\ntrain_merged.shape[1]  # Num of columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:40.154394Z","iopub.execute_input":"2025-06-03T18:33:40.154631Z","iopub.status.idle":"2025-06-03T18:33:40.785459Z","shell.execute_reply.started":"2025-06-03T18:33:40.154613Z","shell.execute_reply":"2025-06-03T18:33:40.784743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Getting orientation, thermopile, and time-of-flight column names\nacc_cols = [f'acc_{axis}' for axis in ['x', 'y', 'z']]\nrot_cols = [f'rot_{axis}' for axis in ['w', 'x', 'y', 'z']]\nthm_cols = [f'thm_{i}' for i in range(1, 6)]\ntof_cols = [f'tof_{i}_v{j}' for i in range(1, 6) for j in range(64)]\n\n# Combine all numeric columns into the final list\nnumerical_cols = [\"age\", \"height_cm\", \"shoulder_to_wrist_cm\", \"elbow_to_wrist_cm\"] + acc_cols + rot_cols + thm_cols + tof_cols\n\n# Getting already labeled column names\nalready_labeled_categorical_cols = [\"adult_child\", \"sex\", \"handedness\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:40.787162Z","iopub.execute_input":"2025-06-03T18:33:40.787376Z","iopub.status.idle":"2025-06-03T18:33:40.792325Z","shell.execute_reply.started":"2025-06-03T18:33:40.787359Z","shell.execute_reply":"2025-06-03T18:33:40.791494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Grouping by sequence_id and then merging remaining columns\ncols = ['sequence_id'] + acc_cols + rot_cols + thm_cols + tof_cols\n\nsummary = (train_merged[cols].groupby('sequence_id').agg(['mean', 'std', 'min', 'max', 'median']))\n\nsummary.columns = ['_'.join(col).strip() for col in summary.columns.values]\n\nsummary = summary.reset_index()\n\nsummary.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:40.793093Z","iopub.execute_input":"2025-06-03T18:33:40.793288Z","iopub.status.idle":"2025-06-03T18:33:49.763466Z","shell.execute_reply.started":"2025-06-03T18:33:40.793274Z","shell.execute_reply":"2025-06-03T18:33:49.762722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Adding summary statistics back to whole grouped dataset\nremaining = train_merged.drop_duplicates('sequence_id')\n\ntrain_df = pd.merge(remaining, summary, on = \"sequence_id\", how = \"left\")\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:49.764312Z","iopub.execute_input":"2025-06-03T18:33:49.764591Z","iopub.status.idle":"2025-06-03T18:33:49.895266Z","shell.execute_reply.started":"2025-06-03T18:33:49.764566Z","shell.execute_reply":"2025-06-03T18:33:49.894620Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Getting new numeric columns\nexclude_cols = set(already_labeled_categorical_cols) | {\"sequence_id\"}\n\nnumerical_cols = [\n    col for col in train_df.select_dtypes(include=[\"number\"]).columns\n    if col not in exclude_cols\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:49.896002Z","iopub.execute_input":"2025-06-03T18:33:49.896222Z","iopub.status.idle":"2025-06-03T18:33:50.017224Z","shell.execute_reply.started":"2025-06-03T18:33:49.896204Z","shell.execute_reply":"2025-06-03T18:33:50.016629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Label encoding the target\nle = LabelEncoder()\ntrain_df[\"gesture\"] = le.fit_transform(train_df[\"gesture\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:50.017981Z","iopub.execute_input":"2025-06-03T18:33:50.018207Z","iopub.status.idle":"2025-06-03T18:33:50.024823Z","shell.execute_reply.started":"2025-06-03T18:33:50.018190Z","shell.execute_reply":"2025-06-03T18:33:50.024033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Storing target gestures\ntarget_classes = [\"Above ear - pull hair\", \"Cheek - pinch skin\", \"Eyebrow - pull hair\", \"Eyelash - pull hair\",\n                  \"Forehead - pull hairline\", \"Forehead - scratch\", \"Neck - pinch skin\", \"Neck - scratch\"]\ntarget_class_indices = [np.where(le.classes_ == cls)[0][0] for cls in target_classes]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:50.025517Z","iopub.execute_input":"2025-06-03T18:33:50.025779Z","iopub.status.idle":"2025-06-03T18:33:50.040647Z","shell.execute_reply.started":"2025-06-03T18:33:50.025752Z","shell.execute_reply":"2025-06-03T18:33:50.039911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing Transformer","metadata":{}},{"cell_type":"code","source":"# Column Transformer\nnum = Pipeline([\n    (\"imputer\", SimpleImputer(strategy=\"median\")),\n    (\"scaler\", StandardScaler())\n])\n\npreprocessor = ColumnTransformer([\n    (\"num\", num, numerical_cols),\n    (\"cat_pass\", \"passthrough\", already_labeled_categorical_cols)\n], remainder = \"drop\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:50.041363Z","iopub.execute_input":"2025-06-03T18:33:50.041619Z","iopub.status.idle":"2025-06-03T18:33:50.055208Z","shell.execute_reply.started":"2025-06-03T18:33:50.041597Z","shell.execute_reply":"2025-06-03T18:33:50.054586Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGBoost Model","metadata":{}},{"cell_type":"code","source":"# Model\nxgboost = xgb.XGBClassifier(\n    subsample = 0.8,\n    reg_lambda = 0.1,\n    reg_alpha = 0,\n    n_estimators = 300,\n    min_child_weight = 3,\n    max_depth = 5,\n    gamma = 0.2,\n    eta = 0.1,\n    colsample_bytree = 0.9,\n    tree_method='hist',\n    device='cuda',\n    objective=\"multi:softprob\",\n    random_state = 42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:50.055976Z","iopub.execute_input":"2025-06-03T18:33:50.056213Z","iopub.status.idle":"2025-06-03T18:33:50.070020Z","shell.execute_reply.started":"2025-06-03T18:33:50.056188Z","shell.execute_reply":"2025-06-03T18:33:50.069400Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pipeline","metadata":{}},{"cell_type":"code","source":"# Pipeline\npipe = Pipeline([\n    (\"preprocessor\", preprocessor),\n    (\"xgboost\", xgboost)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:50.070687Z","iopub.execute_input":"2025-06-03T18:33:50.070893Z","iopub.status.idle":"2025-06-03T18:33:50.084821Z","shell.execute_reply.started":"2025-06-03T18:33:50.070878Z","shell.execute_reply":"2025-06-03T18:33:50.083865Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# X/y Split","metadata":{}},{"cell_type":"code","source":"# Getting X and y\nX = train_df.drop(\"gesture\", axis = 1)\ny = train_df[\"gesture\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:33:58.267014Z","iopub.execute_input":"2025-06-03T18:33:58.267291Z","iopub.status.idle":"2025-06-03T18:33:58.300307Z","shell.execute_reply.started":"2025-06-03T18:33:58.267274Z","shell.execute_reply":"2025-06-03T18:33:58.299723Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Cross Validation","metadata":{}},{"cell_type":"code","source":"%%time\n# Cross validation\nif CROSSVAL:\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n    hierarchical_scores = []\n\n    metric = CompetitionMetric()\n\n    for fold, (train_idx, test_idx) in enumerate(skf.split(X, y)):\n        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_test = y[train_idx], y[test_idx]\n\n        pipe.fit(X_train, y_train)\n        y_pred = pipe.predict(X_test)\n\n        y_test_labels = le.inverse_transform(y_test)\n        y_pred_labels = le.inverse_transform(y_pred)\n\n        solution_df = pd.DataFrame({\n            'id': range(len(y_test_labels)),\n            'gesture': y_test_labels\n        })\n\n        submission_df = pd.DataFrame({\n            'id': range(len(y_pred_labels)),\n            'gesture': y_pred_labels\n        })\n\n        score_fold = metric.calculate_hierarchical_f1(solution_df, submission_df)\n        hierarchical_scores.append(score_fold)\n\n    print(\"Mean Hierarchical F1 Score:\", np.mean(hierarchical_scores))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T18:39:48.551743Z","iopub.execute_input":"2025-06-03T18:39:48.552288Z","iopub.status.idle":"2025-06-03T18:41:59.356716Z","shell.execute_reply.started":"2025-06-03T18:39:48.552266Z","shell.execute_reply":"2025-06-03T18:41:59.356066Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Full Dataset Training","metadata":{}},{"cell_type":"code","source":"%%time\n# Fitting model on whole training set\nif TRAINING:\n    inference_pipeline = pipe.fit(X,y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T12:52:12.463116Z","iopub.execute_input":"2025-06-03T12:52:12.463381Z","iopub.status.idle":"2025-06-03T12:52:56.066600Z","shell.execute_reply.started":"2025-06-03T12:52:12.463362Z","shell.execute_reply":"2025-06-03T12:52:56.065956Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Saving Pipeline and Label Encoder","metadata":{}},{"cell_type":"code","source":"# Saving pipeline and label encoder\nif SAVING:\n    joblib.dump(inference_pipeline, 'pipeline.pkl')\n    joblib.dump(le, 'label_encoder.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T15:45:33.164703Z","iopub.execute_input":"2025-06-02T15:45:33.165139Z","iopub.status.idle":"2025-06-02T15:45:33.223936Z","shell.execute_reply.started":"2025-06-02T15:45:33.165113Z","shell.execute_reply":"2025-06-02T15:45:33.222901Z"}},"outputs":[],"execution_count":null}]}