{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":244130716,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **FOREWORD**\n\nThis baseline kernel builds from the [version-3](https://www.kaggle.com/code/ravi20076/cmi2025-public-baseline-v3) kernel where I use the IMU features and develop a single model to test the CV-LB scores. <br>\nHere, I try and use more models and my newly created ModelTrainer class to train and develop relevant models using IMU features and single multiclass problem. I use boosted trees to solve the problem as on date\n\n**I have simplified the Trainer class to suit the competition. This is better useful than the previous version!**\n\n#### **OTHER REFERENCES**\n\n- https://www.kaggle.com/code/ravi20076/cmi2025-baseline-submission-v1\n- https://www.kaggle.com/code/ravi20076/cmi2025-data-v1\n- https://www.kaggle.com/code/ravi20076/cmi2025-public-imports-v2\n- https://www.kaggle.com/code/ravi20076/cmi2025-public-imports-v1\n- https://www.kaggle.com/code/ravi20076/cmi2025-baseline-train-v1\n- https://www.kaggle.com/code/ravi20076/cmi2025-public-imports-v3\n- https://www.kaggle.com/code/ravi20076/cmi2025-baseline-train-v3","metadata":{}},{"cell_type":"markdown","source":"# **IMPORTS**","metadata":{}},{"cell_type":"code","source":"%%time \n\n!pip install -q -r /kaggle/input/cmi2025-public-imports-v3/req_kaggle.txt \n\nexec( open(f\"/kaggle/input/cmi2025-public-imports-v3/myimports.py\", \"r\").read() )\nexec( open(f\"/kaggle/input/cmi2025-public-imports-v3/myutils.py\", \"r\").read() )\nexec( open(f\"/kaggle/input/cmi2025-public-imports-v3/training.py\", \"r\").read() )\n\nprint()","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **CONFIGURATION**","metadata":{}},{"cell_type":"code","source":"%%time\n\nclass CFG:\n    \"\"\"\n    Configuration class for parameters and CV strategy for tuning and training\n    Some parameters may be unused here as this is a general configuration class\n    \"\"\";\n\n    # Data preparation:-\n    version_nb         = 4\n    model_id           = \"V4_1\"\n    model_label        = \"ML\"\n    test_req           = False\n    test_iter          = 150\n    gpu_switch         = \"ON\" if torch.cuda.is_available() else \"OFF\"\n    state              = 42\n    \n    target             = f'gesture'\n    grouper            = f\"subject\"\n    tgt_mapper         = {\n                            \"Above ear - pull hair\" : 0,\n                            \"Cheek - pinch skin\" : 1,\n                            \"Eyebrow - pull hair\" : 2,\n                            \"Eyelash - pull hair\" : 3, \n                            \"Forehead - pull hairline\" : 4,\n                            \"Forehead - scratch\" : 5,\n                            \"Neck - pinch skin\" : 6, \n                            \"Neck - scratch\" : 7,\n                            \n                            \"Drink from bottle/cup\" : 8,\n                            \"Feel around in tray and pull out an object\" : 9,\n                            \"Glasses on/off\" : 10,\n                            \"Pinch knee/leg skin\" : 11, \n                            \"Pull air toward your face\" : 12,\n                            \"Scratch knee/leg skin\" : 13,\n                            \"Text on phone\" : 14,\n                            \"Wave hello\" : 15,\n                            \"Write name in air\" : 16,\n                            \"Write name on leg\" : 17,\n                        }\n    \n    ip_path            = f\"/kaggle/input/cmi-detect-behavior-with-sensor-data\"\n    op_path            = f\"/kaggle/working\"\n    orig_path          = f\"\"\n    data_path          = f\"\"\n    dtl_preproc_req    = True\n    ftre_plots_req     = True\n    ftre_imp_req       = True\n    nb_orig            = 0\n    orig_all_folds     = False\n\n    # Model Training:-\n    pstprcs_oof        = False\n    pstprcs_train      = False\n    pstprcs_test       = False\n    ML                 = True\n    test_preds_req     = True\n    n_splits           = 5\n    n_repeats          = 1\n    nbrnd_erly_stp     = 100\n    mdlcv_mthd         = 'SGKF'\n    metric_obj         = 'maximize'\n\n    # Global variables for plotting:-\n    grid_specs = {'visible'  : True,\n                  'which'    : 'both',\n                  'linestyle': '--',\n                  'color'    : 'lightgrey',\n                  'linewidth': 0.75\n                 }\n\n    title_specs = {'fontsize'   : 9,\n                   'fontweight' : 'bold',\n                   'color'      : '#992600',\n                  }\n\n\ncv_selector = \\\n{\n \"RKF\"   : RepeatedKFold(n_splits   = CFG.n_splits, n_repeats= CFG.n_repeats, random_state= CFG.state),\n \"RSKF\"  : RepeatedStratifiedKFold(n_splits  = CFG.n_splits, n_repeats= CFG.n_repeats, random_state= CFG.state),\n \"SKF\"   : StratifiedKFold(n_splits   = CFG.n_splits, shuffle = True, random_state= CFG.state),\n \"KF\"    : KFold(n_splits = CFG.n_splits, shuffle = True, random_state= CFG.state),\n \"GKF\"   : GroupKFold(n_splits   = CFG.n_splits),\n \"SGKF\"  : StratifiedGroupKFold(n_splits = CFG.n_splits, shuffle = True, random_state= CFG.state),\n}\n\ncollect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **KEY CONFIG PARAMETERS**\n\n|Configuration parameter| Explanation| Data type| Sample values |  \n| ---------------------- | ------------------------------- | --------------------- | --------------- |\n| version_nb    | Version Number | int | 1 | \n| model_id      | Model ID    | string | V1_1 | \n| model_label   | Model Label | string | ML | \n| test_req      | Test Required| bool | True / False | \n| test_iter     | Test case iterations for models | int | 50 |\n| gpu_switch      | Do we need GPU support | bool | True / False |\n| state           | Random state | int | 42 |\n| target          | Target column | str |  |\n| grouper         | CV grouper column | str |  |\n| ip_path, op_path | Data paths  | str | |\n| pstprcs_* | Do we need post-processing  | bool |True / False |\n| ML| Do we need machine learning models  | bool |True / False |\n| test_preds_req| Do we need test set predictions (training in inference kernel)  | bool |True / False |\n| n_splits/ n_repeats | N-splits and repeats for CV scheme | int | 3/5/10|\n| nbrnd_erly_stp | Early stopping rounds | int | 40|\n| mdlcv_mthd | Model CV method | str | RSKF|\n| ensemble_req | Do we need ensemble | bool | True / False |\n| metric_obj   | Metric direction | str | minimize/ maximize |","metadata":{}},{"cell_type":"markdown","source":"# **PREPROCESSING**\n\n\nWe load the datasets, concoct the relevant features and initialize the CV folds here","metadata":{}},{"cell_type":"code","source":"%%writefile myfe.py\n\ndef make_ftre(sequence : pl.DataFrame, demographics : pl.DataFrame) :\n    \"\"\"\n    Makes aggregate columns for the model\n    Source - https://www.kaggle.com/code/farisalahmdi/lgbm-inference\n    \"\"\"\n    \n    agg_exprs = []\n    stat_cols = ['acc_x','acc_y','acc_z','rot_w','rot_x','rot_y','rot_z']\n    \n    for c in stat_cols:\n        agg_exprs += [\n            pl.col(c).mean().alias(f\"{c}_mean\"),\n            pl.col(c).std().alias(f\"{c}_std\"),\n            pl.col(c).var().alias(f\"{c}_var\"),\n            pl.col(c).quantile(0.25).alias(f\"{c}_q25\"),\n            pl.col(c).median().alias(f\"{c}_q50\"),\n            pl.col(c).quantile(0.75).alias(f\"{c}_q75\"),\n            pl.col(c).max().alias(f\"{c}_max\"),\n            pl.col(c).min().alias(f\"{c}_min\"),\n            pl.col(c).first().alias(f\"{c}_first\"),\n            pl.col(c).last().alias(f\"{c}_last\"),\n            pl.col(c).quantile(0.25, \"nearest\").alias(f\"{c}_t25\"),\n            pl.col(c).quantile(0.75, \"nearest\").alias(f\"{c}_t75\"),\n            (pl.col(c).last() - pl.col(c).first()).alias(f\"{c}_delta\"),\n            pl.corr(\"sequence_counter\", c).alias(f\"{c}_corr_time\"),\n            pl.col(c).diff().mean().alias(f\"{c}_diff_mean\"),\n            pl.col(c).diff().std().alias(f\"{c}_diff_std\"),\n            pl.col(c).skew().alias(f\"{c}_skew\"),\n            pl.col(c).kurtosis().alias(f\"{c}_kurt\"),\n            pl.col(c).diff().abs().gt(0).sum().alias(f\"{c}_n_changes\")\n        ]\n        \n        agg_exprs += [\n                pl.when(pl.col(\"sequence_counter\") < 0.1 * pl.max(\"sequence_counter\"))\n                  .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg1_mean\"),\n                pl.when(pl.col(\"sequence_counter\") > 0.9 * pl.max(\"sequence_counter\"))\n                  .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg3_mean\"),\n        ]\n    \n    return \\\n    (\n        sequence.\n        group_by(pl.col([\"sequence_id\", \"subject\"]), maintain_order=True).\n        agg(agg_exprs).\n        select(pl.all().shrink_dtype()).\n        join(\n            demographics.select(pl.all().shrink_dtype()),\n            how = \"left\",\n            on = [\"subject\"],\n        ).\n        drop([\"subject\"], strict = False).\n        to_pandas()\n    )","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time \n\nexec( open(f\"myfe.py\", \"r\").read() )\n\ntrain    = pl.read_csv(f\"{CFG.ip_path}/train.csv\")\ntraind   = pl.read_csv(f\"{CFG.ip_path}/train_demographics.csv\")\ntest     = pl.read_csv(f\"{CFG.ip_path}/test.csv\")\ntestd    = pl.read_csv(f\"{CFG.ip_path}/test_demographics.csv\")\n\nsel_cols = [ c for c in test.collect_schema().names() if \"thm\" not in c and \"tof\" not in c ]\n\nXtrain  = make_ftre(train.select(pl.col(sel_cols)), traind)\nXtest   = make_ftre(test.select(pl.col(sel_cols)),  testd)\n\nYtrain             = train.select(pl.col([\"sequence_id\", CFG.grouper, CFG.target])).unique().to_pandas()\nYtrain[CFG.target] = Ytrain[CFG.target].map(CFG.tgt_mapper).astype(np.int8)\nYtrain             = Xtrain[[\"sequence_id\"]].merge(Ytrain, how = \"inner\", on = [\"sequence_id\"])\nytrain             = Ytrain[CFG.target]\n\nygrp = np.zeros(len(Xtrain))\ncv   = cv_selector[CFG.mdlcv_mthd]\n\nfor fold_nb, (train_idx, dev_idx) in tqdm( \n    enumerate(cv.split(Xtrain, Ytrain[CFG.target], Ytrain[CFG.grouper] )) \n) :\n    ygrp[dev_idx] = fold_nb\n\ncv = PredefinedSplit(ygrp)\nprint(\n    f\"\\n\\n---> Shape = {Xtrain.shape} {ytrain.shape} {ygrp.shape} {Xtest.shape}\"\n)\n\n_ = utils.CleanMemory()\nprint()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **MODEL TRAINING**\n\nLet's start off with simple baseline single models and blend them later","metadata":{}},{"cell_type":"code","source":"%%time \n\nMdl_Master = \\\n{ \n f'LGBM1C'  : [LGBMC(**{'device'             : \"gpu\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n                        'objective'          : \"multiclass\",\n                        \"n_estimators\"       : 10000 if CFG.test_req == False else CFG.test_iter, \n                        \"max_depth\"          : 8,\n                        \"learning_rate\"      : 0.025,\n                        \"colsample_bytree\"   : 0.55,               \n                        \"n_jobs\"             : -1,\n                        \"num_leaves\"         : 75,\n                        \"random_state\"       : CFG.state,\n                        \"reg_alpha\"          : 0.001,\n                        \"reg_lambda\"         : 0.001,\n                        \"subsample\"          : 0.40,\n                        \"verbosity\"          : -1,\n                      }\n                   ),\n               {\"callbacks\" : [log_evaluation(0), early_stopping(CFG.nbrnd_erly_stp, verbose = False)]},\n              ],\n\n f'LGBM2C'  : [LGBMC(**{\"data_sample_strategy\"   : \"goss\",\n                        \"n_estimators\"           : 10000 if CFG.test_req == False else CFG.test_iter, \n                        \"max_depth\"              : 7, \n                        'device'                 : \"gpu\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n                        \"learning_rate\"          : 0.025,\n                        \"colsample_bytree\"       : 0.55,\n                        \"min_child_samples\"      : 32,\n                        \"n_jobs\"                 : -1,\n                        \"num_leaves\"             : 60,\n                        \"random_state\"           : CFG.state,\n                        \"reg_alpha\"              : 0.001,\n                        \"reg_lambda\"             : 0.001,\n                        \"subsample\"              : 0.21,\n                        \"verbosity\"              : -1,\n                     }\n                   ),\n               {\"callbacks\" : [log_evaluation(0), early_stopping(CFG.nbrnd_erly_stp, verbose = False)]},\n              ],\n\n f'CB1C'    : [CBC(**{'task_type'             : \"GPU\" if CFG.gpu_switch == \"ON\" else \"CPU\",\n                      'loss_function'         : \"MultiClass\",\n                      'learning_rate'         : 0.04,\n                      'iterations'            : 10000 if CFG.test_req == False else CFG.test_iter,\n                      'max_depth'             : 8,\n                      'min_data_in_leaf'      : 12,\n                      'colsample_bylevel'     : 0.30 if CFG.gpu_switch == \"OFF\" else None,\n                      'l2_leaf_reg'           : 0.35,\n                      'leaf_estimation_method': \"Newton\",\n                      'verbose'               : 0,\n                      'random_state'          : CFG.state,\n                      'early_stopping_rounds' : CFG.nbrnd_erly_stp if CFG.nbrnd_erly_stp > 0 else None,\n                    }\n                 ),\n               {\"verbose\" : 0 },\n              ],\n\n f'XGB1C'  : [  XGBC(**{'device'                : \"cuda:0\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n                        'objective'             : \"multi:softprob\",\n                        \"n_estimators\"          : 10000 if CFG.test_req == False else CFG.test_iter, \n                        \"max_depth\"             : 8,\n                        \"learning_rate\"         : 0.025,\n                        \"colsample_bytree\"      : 0.55,               \n                        \"n_jobs\"                : -1,\n                        \"random_state\"          : CFG.state,\n                        \"reg_alpha\"             : 0.001,\n                        \"reg_lambda\"            : 0.001,\n                        \"verbosity\"             : 0,\n                        \"enable_categorical\"    : True,\n                        'early_stopping_rounds' : CFG.nbrnd_erly_stp if CFG.nbrnd_erly_stp > 0 else None,\n                      }\n                   ),\n               {\"verbose\" : 0},\n              ],\n}\n\n# Initializing model outputs\nOOF_Preds    = []\nFittedModels = {}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time \n\ndrop_cols = \\\n[\"Source\", \"id\", \"Id\", \"Label\", CFG.target, \"fold_nb\", \"sequence_id\", CFG.grouper]\n\nfor method, (mymodel, fit_params) in tqdm(Mdl_Master.items()):\n\n    md = \\\n    ModelTrainer(\n        problem_type     = \"multiclass\",\n        drop_cols        = drop_cols, \n        len_train        = Xtrain.shape[0],\n        test_preds_req   = CFG.test_preds_req, \n        bin_cutoff       = 0.50,\n    )\n\n    (fitted_models, oof_preds, test_preds) =  \\\n    md.fit_predict(\n        Xtrain,\n        ytrain,\n        Xtest,\n        ygrp,\n        method,\n        mymodel,\n        cat_cols = None,\n        **fit_params, \n    )\n\n    OOF_Preds.append(oof_preds)\n    FittedModels[method] = fitted_models\n\n    del fitted_models, oof_preds, test_preds\n    print()\n    collect()\n\n_ = utils.CleanMemory()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **ENSEMBLE**\n\nWe simple average out the multiclass predictions to elicit a final value","metadata":{}},{"cell_type":"code","source":"%%time \n\nscore, bscore, mscore = \\\nutils.ScoreMetric(\n    ytrain.values.flatten(),\n    pd.concat(\n        OOF_Preds, axis=0, ignore_index = False\n    ).\n    groupby(level = 0).\n    mean().\n    idxmax(axis=1).\n    to_numpy().\n    flatten()\n)\n\nPrintColor(\n    f\"\\n---> Ensemble Score = {score:,.8f} | Binary = {bscore:,.8f} | Multiclass = {mscore:,.8f}\\n\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **CLOSURE**","metadata":{}},{"cell_type":"code","source":"joblib.dump(\n    FittedModels, \n    f\"{CFG.op_path}/FittedModels{CFG.model_label}{CFG.model_id}.joblib\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}