{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":242561727,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is the training notebook, if you want the prediction notebook...\n\nMy inference notebook is at [CMI 2025 - Inference](https://www.kaggle.com/code/coreymichaud/cmi-2025-inference)\n\nThis notebook is a \"copy\" of my other XGBoost notebook, however in this one I use Weights & Biases for model logging and getting the hyperparameters with W&B Sweep.","metadata":{}},{"cell_type":"code","source":"# Libraries\nimport os\n\nimport pandas as pd\nimport numpy as np\nimport polars as pl\n\nimport joblib\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import train_test_split\n\nfrom metric import score, CompetitionMetric  # metric given by the competition\n\nimport xgboost as xgb\n\nimport wandb\nfrom wandb.integration.xgboost import WandbCallback\nimport os\nfrom kaggle_secrets import UserSecretsClient\n\n# OS environment\nos.environ[\"WANDB_SILENT\"] = \"true\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:00.501129Z","iopub.execute_input":"2025-06-19T12:13:00.501450Z","iopub.status.idle":"2025-06-19T12:13:06.022615Z","shell.execute_reply.started":"2025-06-19T12:13:00.501431Z","shell.execute_reply":"2025-06-19T12:13:06.022045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CONFIG\nSWEEP = 0\nSAVE_MODEL = 1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:06.023263Z","iopub.execute_input":"2025-06-19T12:13:06.023677Z","iopub.status.idle":"2025-06-19T12:13:06.027189Z","shell.execute_reply.started":"2025-06-19T12:13:06.023658Z","shell.execute_reply":"2025-06-19T12:13:06.026424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Getting Kaggle secrets\nuser_secrets = UserSecretsClient()\n\n# Getting wandb api key\napi_key = user_secrets.get_secret(\"wandb_api_key\")\n\n# Logging into wandb\nwandb.login(key = api_key)  # prints 'True' if api key works","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:06.028761Z","iopub.execute_input":"2025-06-19T12:13:06.028957Z","iopub.status.idle":"2025-06-19T12:13:12.376209Z","shell.execute_reply.started":"2025-06-19T12:13:06.028943Z","shell.execute_reply":"2025-06-19T12:13:12.375476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Importing training data\ntrain = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\ntrain_demographics = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:12.377039Z","iopub.execute_input":"2025-06-19T12:13:12.377428Z","iopub.status.idle":"2025-06-19T12:13:41.429917Z","shell.execute_reply.started":"2025-06-19T12:13:12.377410Z","shell.execute_reply":"2025-06-19T12:13:41.429341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combining train with train_demographics ... should have 348 columns if successful\ntrain_merged = pd.merge(train, train_demographics, on = \"subject\", how = \"left\")\ntrain_merged.shape[1]  # Num of columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:41.430609Z","iopub.execute_input":"2025-06-19T12:13:41.430807Z","iopub.status.idle":"2025-06-19T12:13:42.049613Z","shell.execute_reply.started":"2025-06-19T12:13:41.430791Z","shell.execute_reply":"2025-06-19T12:13:42.049007Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Getting orientation, thermopile, and time-of-flight column names\nacc_cols = [f'acc_{axis}' for axis in ['x', 'y', 'z']]\nrot_cols = [f'rot_{axis}' for axis in ['w', 'x', 'y', 'z']]\nthm_cols = [f'thm_{i}' for i in range(1, 6)]\ntof_cols = [f'tof_{i}_v{j}' for i in range(1, 6) for j in range(64)]\n\n# Combine all numeric columns into the final list\nnumerical_cols = [\"age\", \"height_cm\", \"shoulder_to_wrist_cm\", \"elbow_to_wrist_cm\"] + acc_cols + rot_cols + thm_cols + tof_cols\n\n# Getting already labeled column names\nalready_labeled_categorical_cols = [\"adult_child\", \"sex\", \"handedness\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:42.050330Z","iopub.execute_input":"2025-06-19T12:13:42.050589Z","iopub.status.idle":"2025-06-19T12:13:42.055513Z","shell.execute_reply.started":"2025-06-19T12:13:42.050559Z","shell.execute_reply":"2025-06-19T12:13:42.054770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Grouping by sequence_id and then merging remaining columns\ncols = ['sequence_id'] + acc_cols + rot_cols + thm_cols + tof_cols\n\nsummary = (train_merged[cols].groupby('sequence_id').agg(['mean', 'std', 'min', 'max', 'median']))\n\nsummary.columns = ['_'.join(col).strip() for col in summary.columns.values]\n\nsummary = summary.reset_index()\n\nsummary.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:42.056278Z","iopub.execute_input":"2025-06-19T12:13:42.056475Z","iopub.status.idle":"2025-06-19T12:13:50.697315Z","shell.execute_reply.started":"2025-06-19T12:13:42.056458Z","shell.execute_reply":"2025-06-19T12:13:50.696708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Adding summary statistics back to whole grouped dataset\nremaining = train_merged.drop_duplicates('sequence_id')\n\ntrain_df = pd.merge(remaining, summary, on = \"sequence_id\", how = \"left\")\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:50.697972Z","iopub.execute_input":"2025-06-19T12:13:50.698186Z","iopub.status.idle":"2025-06-19T12:13:50.841896Z","shell.execute_reply.started":"2025-06-19T12:13:50.698169Z","shell.execute_reply":"2025-06-19T12:13:50.841345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Getting new numeric columns\nexclude_cols = set(already_labeled_categorical_cols) | {\"sequence_id\"}\n\nnumerical_cols = [\n    col for col in train_df.select_dtypes(include=[\"number\"]).columns\n    if col not in exclude_cols\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:50.843911Z","iopub.execute_input":"2025-06-19T12:13:50.844162Z","iopub.status.idle":"2025-06-19T12:13:50.957933Z","shell.execute_reply.started":"2025-06-19T12:13:50.844145Z","shell.execute_reply":"2025-06-19T12:13:50.957358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Label encoding the target\nle = LabelEncoder()\ntrain_df[\"gesture\"] = le.fit_transform(train_df[\"gesture\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:50.958647Z","iopub.execute_input":"2025-06-19T12:13:50.958920Z","iopub.status.idle":"2025-06-19T12:13:50.965352Z","shell.execute_reply.started":"2025-06-19T12:13:50.958893Z","shell.execute_reply":"2025-06-19T12:13:50.964788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Storing target gestures\ntarget_classes = [\"Above ear - pull hair\", \"Cheek - pinch skin\", \"Eyebrow - pull hair\", \"Eyelash - pull hair\",\n                  \"Forehead - pull hairline\", \"Forehead - scratch\", \"Neck - pinch skin\", \"Neck - scratch\"]\ntarget_class_indices = [np.where(le.classes_ == cls)[0][0] for cls in target_classes]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:50.965958Z","iopub.execute_input":"2025-06-19T12:13:50.966179Z","iopub.status.idle":"2025-06-19T12:13:50.980438Z","shell.execute_reply.started":"2025-06-19T12:13:50.966156Z","shell.execute_reply":"2025-06-19T12:13:50.979727Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing Transformer","metadata":{}},{"cell_type":"code","source":"# Column Transformer\nnum = Pipeline([\n    (\"imputer\", SimpleImputer(strategy=\"median\")),\n    (\"scaler\", StandardScaler())\n])\n\npreprocessor = ColumnTransformer([\n    (\"num\", num, numerical_cols),\n    (\"cat_pass\", \"passthrough\", already_labeled_categorical_cols)\n], remainder = \"drop\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:50.981193Z","iopub.execute_input":"2025-06-19T12:13:50.981410Z","iopub.status.idle":"2025-06-19T12:13:50.995805Z","shell.execute_reply.started":"2025-06-19T12:13:50.981386Z","shell.execute_reply":"2025-06-19T12:13:50.995094Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# X/y Split","metadata":{}},{"cell_type":"code","source":"# Getting X and y\nX = train_df.drop(\"gesture\", axis = 1)\ny = train_df[\"gesture\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:50.996558Z","iopub.execute_input":"2025-06-19T12:13:50.997029Z","iopub.status.idle":"2025-06-19T12:13:51.039936Z","shell.execute_reply.started":"2025-06-19T12:13:50.997013Z","shell.execute_reply":"2025-06-19T12:13:51.039106Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Sweeps","metadata":{}},{"cell_type":"code","source":"if SWEEP:\n    def main():\n        with wandb.init() as run:\n    \n            # Split dataset\n            X_temp, X_test, y_temp, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n            X_train, X_val, y_train, y_val = train_test_split(X_temp, y_temp, test_size=0.25, random_state=42,stratify=y_temp)\n            # 60% train, 20% val, 20% test\n        \n            # Preprocessing\n            X_train_trans = preprocessor.fit_transform(X_train)\n            X_val_trans = preprocessor.transform(X_val)\n            X_test_trans = preprocessor.transform(X_test)\n        \n            # Initialize XGBoost with wandb.config hyperparameters\n            xgboost = xgb.XGBClassifier(\n                subsample=wandb.config.subsample,\n                reg_lambda=wandb.config.reg_lambda,\n                reg_alpha=wandb.config.reg_alpha,\n                min_child_weight=wandb.config.min_child_weight,\n                max_depth=wandb.config.max_depth,\n                gamma=wandb.config.gamma,\n                eta=wandb.config.eta,\n                colsample_bytree=wandb.config.colsample_bytree,\n                n_estimators=1000,\n                random_state=42,\n                verbosity=0,\n                tree_method='hist',\n                device='cuda',\n                objective=\"multi:softprob\",\n                eval_metric=\"mlogloss\",\n                callbacks=[WandbCallback()],\n                early_stopping_rounds = 15\n            )\n        \n            # Fit with validation set, no early stopping, use wandb callback to log metrics every iteration\n            xgboost.fit(\n                X_train_trans, y_train,\n                eval_set=[(X_train_trans, y_train), (X_val_trans, y_val)],\n                verbose = 0\n            )\n        \n            # Predict and evaluate final test metric\n            y_pred = xgboost.predict(X_test_trans)\n            y_test_labels = le.inverse_transform(y_test)\n            y_pred_labels = le.inverse_transform(y_pred)\n        \n            solution_df = pd.DataFrame({'id': range(len(y_test_labels)), 'gesture': y_test_labels})\n            submission_df = pd.DataFrame({'id': range(len(y_pred_labels)), 'gesture': y_pred_labels})\n        \n            metric = CompetitionMetric()\n            hierarchical_f1 = metric.calculate_hierarchical_f1(solution_df, submission_df)\n        \n            wandb.log({\"hierarchical_f1\": hierarchical_f1,\n                      \"best_iteration\": xgboost.best_iteration})\n    \n    # Sweep config (adjust metric to the validation metric logged by callback)\n    sweep_config = dict(\n        method=\"bayes\",\n        name=\"bayes-optimization-xgboost\",\n        metric={'goal': 'minimize', 'name': 'validation_1-mlogloss'},  # or 'hierarchical_f1' if you prefer\n        early_terminate=dict(\n            type='hyperband',\n            min_iter=20,\n            eta=2\n        ),\n        parameters=dict(\n            subsample=dict(min=0.5, max=1.0, distribution='uniform'),\n            reg_lambda=dict(min=0.1, max=50.0, distribution='uniform'),\n            reg_alpha=dict(min=0.0, max=10.0, distribution='uniform'),\n            min_child_weight=dict(min=1, max=10, distribution='int_uniform'),\n            max_depth=dict(min=4, max=16, distribution='int_uniform'),\n            gamma=dict(min=0.0, max=5.0, distribution='uniform'),\n            eta=dict(min=0.01, max=0.3, distribution='uniform'),\n            colsample_bytree=dict(min=0.5, max=1.0, distribution='uniform')\n        )\n    )\n        \n    # Run the sweep\n    sweep_id = wandb.sweep(sweep=sweep_config, entity='coreymichaud-projects', project='CMI 2025 Kaggle')\n    wandb.agent(sweep_id, function=main, count=500)\n    # ^ this may have runs that give errors due to running out of memory. in that case, the run ends but the rest keep going, so the code\n    # will keep running :)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:51.040681Z","iopub.execute_input":"2025-06-19T12:13:51.040860Z","iopub.status.idle":"2025-06-19T12:13:51.050894Z","shell.execute_reply.started":"2025-06-19T12:13:51.040846Z","shell.execute_reply":"2025-06-19T12:13:51.050167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if SWEEP:\n#     def main():\n#         run = wandb.init()\n        \n#         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    \n#         xgboost = xgb.XGBClassifier(\n#             subsample=wandb.config.subsample,\n#             reg_lambda=wandb.config.reg_lambda,\n#             reg_alpha=wandb.config.reg_alpha,\n#             min_child_weight=wandb.config.min_child_weight,\n#             max_depth=wandb.config.max_depth,\n#             gamma=wandb.config.gamma,\n#             eta=wandb.config.eta,\n#             colsample_bytree=wandb.config.colsample_bytree,\n#             n_estimators=wandb.config.n_estimators,\n#             random_state=42,\n#             verbosity=0,\n#             tree_method='hist',\n#             device='cuda',\n#             objective=\"multi:softprob\"\n#         )\n    \n#         # Pipeline\n#         pipe = Pipeline([\n#             (\"preprocessor\", preprocessor),\n#             (\"xgboost\", xgboost)\n#         ])\n    \n#         # Fit model with early stopping\n#         pipe.fit(X_train, y_train)\n    \n#         # Predict and evaluate\n#         y_pred = pipe.predict(X_test)\n#         y_test_labels = le.inverse_transform(y_test)\n#         y_pred_labels = le.inverse_transform(y_pred)\n    \n#         solution_df = pd.DataFrame({'id': range(len(y_test_labels)), 'gesture': y_test_labels})\n#         submission_df = pd.DataFrame({'id': range(len(y_pred_labels)), 'gesture': y_pred_labels})\n    \n#         metric = CompetitionMetric()\n#         hierarchical_f1 = metric.calculate_hierarchical_f1(solution_df, submission_df)\n    \n#         # Log metrics\n#         wandb.log({\n#             \"hierarchical_f1\": hierarchical_f1\n#         })\n    \n#         run.finish()\n    \n#     # Sweep configuration\n#     sweep_config = dict(\n#         method=\"bayes\",\n#         name=\"bayes-optimization-xgboost\",\n#         metric={'goal': 'maximize', 'name': 'hierarchical_f1'},\n#         parameters=dict(\n#             subsample=dict(min=0.5, max=1.0, distribution='uniform'),\n#             reg_lambda=dict(min=0.1, max=10.0, distribution='log_uniform'),\n#             reg_alpha=dict(min=0.0, max=5.0, distribution='log_uniform'),\n#             min_child_weight=dict(min=1, max=10, distribution='int_uniform'),\n#             max_depth=dict(min=4, max=16, distribution='int_uniform'),\n#             gamma=dict(min=0.0, max=5.0, distribution='uniform'),\n#             eta=dict(min=0.01, max=0.3, distribution='log_uniform'),\n#             colsample_bytree=dict(min=0.5, max=1.0, distribution='uniform'),\n#             n_estimators=dict(min=50, max=350, distribution='int_uniform')\n#         )\n#     )\n    \n#     # Run the sweep\n#     sweep_id = wandb.sweep(sweep=sweep_config, entity='coreymichaud-projects', project='CMI 2025 Kaggle')\n#     wandb.agent(sweep_id, function=main, count=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:13:51.051684Z","iopub.execute_input":"2025-06-19T12:13:51.051845Z","iopub.status.idle":"2025-06-19T12:13:51.069727Z","shell.execute_reply.started":"2025-06-19T12:13:51.051832Z","shell.execute_reply":"2025-06-19T12:13:51.069200Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Full Dataset Training","metadata":{}},{"cell_type":"code","source":"if SAVE_MODEL:\n    \n    # Model parameters\n    params = dict(\n        subsample = 0.7879940858696578,\n        reg_lambda = 23.324112198930496,\n        reg_alpha = 0.5523929411331463,\n        n_estimators = 375,\n        min_child_weight = 8,\n        max_depth = 9,\n        gamma = 0.508831354869273,\n        eta = 0.15668085003381013,\n        colsample_bytree = 0.5252982328734005,\n        random_state = 42,\n        tree_method='hist',\n        device='cuda',\n        objective=\"multi:softprob\"\n    )\n    \n    # Opening wandb run\n    with wandb.init(entity = \"coreymichaud-projects\", project = \"CMI 2025 Kaggle\", config = params) as run:\n        \n        # Model with best parameters\n        xgboost = xgb.XGBClassifier(**params)\n        \n        # Pipeline\n        pipe = Pipeline([\n            (\"preprocessor\", preprocessor),\n            (\"xgboost\", xgboost)\n        ])\n        \n        # Fitting model on whole training set\n        inference_pipeline = pipe.fit(X,y)\n\n        # Function to save model as artifact for model logging\n        def save_artifact(model, model_name, filename):\n            joblib.dump(model, filename)\n            artifact = wandb.Artifact(name = model_name, type = \"model\")\n            artifact.add_file(filename)\n            run.log_artifact(artifact)\n        \n        save_artifact(inference_pipeline, \"xgbclassifier\", \"xgbclassifier.joblib\")  # PyTorch has model.save and the file would be a .pth file\n\n        # Saving model and label encoder within this notebook so I can use in the inference notebook\n        joblib.dump(inference_pipeline, 'pipeline.pkl')  # this is being saved... again... because the other nb has .pkl set up already\n        joblib.dump(le, 'label_encoder.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-19T12:20:28.716351Z","iopub.execute_input":"2025-06-19T12:20:28.716915Z","iopub.status.idle":"2025-06-19T12:21:17.873180Z","shell.execute_reply.started":"2025-06-19T12:20:28.716895Z","shell.execute_reply":"2025-06-19T12:21:17.872620Z"}},"outputs":[],"execution_count":null}]}