{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":10097128,"sourceType":"datasetVersion","datasetId":6226248},{"sourceId":242895646,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# References\n- https://www.kaggle.com/code/ravi20076/cmi2025-public-baseline-v3?scriptVersionId=244089160\n- https://www.kaggle.com/code/yekenot/cibmtr-tabm-nn/notebook\n\n## Enhanced Feature Engineering Results\n\n### Performance Improvement\n**Cross-Validation Results from this notebook V1:**\n- **TabM**: 0.686638 → **0.716331** (+0.030)\n- **CatBoost**: 0.691123 → **0.717714** (+0.026)\n- **Public LB**: 0.70 → **0.72** (+0.02)\n\n### Added Features (~65 new features, 155→220 dimensions)\n\n#### 1. BFRB-Specific Movement Patterns (49 features)\n- **Peak detection**: Count significant changes for repetitive behavior detection\n- **Movement intensity**: Mean absolute deviation to measure action force\n- **Range normalization**: Signal range relative to standard deviation\n- **Acceleration analysis**: Second derivative for sudden movement detection\n- **Movement consistency**: Coefficient of variation for pattern regularity\n- **Energy measure**: Sum of squared values for motion intensity\n\n#### 2. Multi-Axis Interaction Features (15 features)\n- **3D magnitude**: Combined acceleration/rotation vector magnitudes\n- **Axis dominance**: Which axis contributes most to movement\n- **Cross-correlations**: Inter-axis relationships (acc_xy, acc_yz, etc.)\n- **Rotation-acceleration coupling**: Correlation between rotation and linear motion\n\n#### 3. Temporal Pattern Analysis (14 features)\n- **Sequence characteristics**: Length and count metrics\n- **Early vs late behavior**: Variance ratio between sequence phases\n- **Movement intensity rates**: Change density over time\n- **Motion complexity**: Total variation normalized by sequence length\n- **Smoothness metrics**: Jerk analysis for movement quality assessment\n\nThese enhancements specifically target **repetitive and directional patterns** characteristic of BFRB behaviors, leading to improved discrimination between target and non-target gestures.","metadata":{}},{"cell_type":"markdown","source":"### If you find this notebook useful, please upvote this and the original notebooks!","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --no-deps --find-links=/kaggle/input/tabm-tabular-dl-library rtdl_num_embeddings==0.0.11\n!pip install --no-index --no-deps --find-links=/kaggle/input/tabm-tabular-dl-library tabm==0.0.1.dev0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:10.972320Z","iopub.execute_input":"2025-06-18T04:47:10.972524Z","iopub.status.idle":"2025-06-18T04:47:14.784259Z","shell.execute_reply.started":"2025-06-18T04:47:10.972507Z","shell.execute_reply":"2025-06-18T04:47:14.783303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q polars==1.29.0      --no-index --find-links=/kaggle/input/cmi2025-public-imports-v2/packages\n!pip install -q lightgbm==4.6.0     --no-index --find-links=/kaggle/input/cmi2025-public-imports-v2/packages\n!pip install -q scikit-learn==1.6.1 --no-index --find-links=/kaggle/input/cmi2025-public-imports-v2/packages","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:14.785405Z","iopub.execute_input":"2025-06-18T04:47:14.785724Z","iopub.status.idle":"2025-06-18T04:47:29.980869Z","shell.execute_reply.started":"2025-06-18T04:47:14.785681Z","shell.execute_reply":"2025-06-18T04:47:29.980189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Import & Setup","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nimport joblib\nfrom tqdm.notebook import tqdm\nfrom gc import collect\nimport warnings\nimport math\nimport random\nfrom typing import Optional, Tuple\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\n\nfrom sklearn.base import clone, BaseEstimator\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler, OrdinalEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold, PredefinedSplit\nfrom sklearn.metrics import f1_score\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.ensemble import BaggingClassifier as BC\n\n# LightGBM and CatBoost\nfrom lightgbm import LGBMClassifier as LGBMC, log_evaluation, early_stopping\nfrom catboost import CatBoostClassifier as CBC\n\n# TabM imports\nimport sys\nsys.path.append('/kaggle/input/tabm-tabular-dl-library')\nfrom tabm_reference import Model, make_parameter_groups\nimport rtdl_num_embeddings\n\nwarnings.filterwarnings('ignore')\n\ndef seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.benchmark = True\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:29.982916Z","iopub.execute_input":"2025-06-18T04:47:29.983149Z","iopub.status.idle":"2025-06-18T04:47:38.895443Z","shell.execute_reply.started":"2025-06-18T04:47:29.983126Z","shell.execute_reply":"2025-06-18T04:47:38.894915Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration & Constants","metadata":{}},{"cell_type":"code","source":"target = \"gesture\"\ngrouper = \"subject\"\n\nmapper = {\n    \"Above ear - pull hair\": 0,\n    \"Cheek - pinch skin\": 1,\n    \"Eyebrow - pull hair\": 2,\n    \"Eyelash - pull hair\": 3, \n    \"Forehead - pull hairline\": 4,\n    \"Forehead - scratch\": 5,\n    \"Neck - pinch skin\": 6, \n    \"Neck - scratch\": 7,\n    \"Drink from bottle/cup\": 8,\n    \"Feel around in tray and pull out an object\": 9,\n    \"Glasses on/off\": 10,\n    \"Pinch knee/leg skin\": 11, \n    \"Pull air toward your face\": 12,\n    \"Scratch knee/leg skin\": 13,\n    \"Text on phone\": 14,\n    \"Wave hello\": 15,\n    \"Write name in air\": 16,\n    \"Write name on leg\": 17,\n}\n\ndef ScoreMetric(ytrue, ypreds) -> tuple:\n    \"\"\"CMI competition scoring metric\"\"\"\n    bscore = f1_score(\n        np.where(ytrue <= 7, 1, 0),\n        np.where(ypreds <= 7, 1, 0),\n        zero_division=0.0,\n    )\n    mscore = f1_score(\n        np.where(ytrue <= 7, ytrue, 99),\n        np.where(ypreds <= 7, ypreds, 99),\n        average=\"macro\", \n        zero_division=0.0,\n    )\n    return (0.5 * (bscore + mscore), bscore, mscore)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:38.898441Z","iopub.execute_input":"2025-06-18T04:47:38.898637Z","iopub.status.idle":"2025-06-18T04:47:38.904552Z","shell.execute_reply.started":"2025-06-18T04:47:38.898592Z","shell.execute_reply":"2025-06-18T04:47:38.903859Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Enhanced Feature Engineering","metadata":{}},{"cell_type":"code","source":"def make_ftre_unified_enhanced(sequence: pl.DataFrame, demographics: pl.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Enhanced feature extraction with BFRB-specific patterns\n    Total: ~220 dimensions (original 155 + 65 new features)\n    Designed to work with existing codebase architecture\n    \"\"\"\n    \n    agg_exprs = []\n    stat_cols = ['acc_x','acc_y','acc_z','rot_w','rot_x','rot_y','rot_z']\n    \n    # === EXISTING FEATURES (155 dimensions) - Keep unchanged ===\n    for c in stat_cols:\n        # Original features from the base implementation\n        agg_exprs += [\n            pl.col(c).mean().alias(f\"{c}_mean\"),\n            pl.col(c).std().alias(f\"{c}_std\"),\n            pl.col(c).var().alias(f\"{c}_var\"),\n            pl.col(c).quantile(0.25).alias(f\"{c}_q25\"),\n            pl.col(c).median().alias(f\"{c}_q50\"),\n            pl.col(c).quantile(0.75).alias(f\"{c}_q75\"),\n            pl.col(c).max().alias(f\"{c}_max\"),\n            pl.col(c).min().alias(f\"{c}_min\"),\n            pl.col(c).first().alias(f\"{c}_first\"),\n            pl.col(c).last().alias(f\"{c}_last\"),\n            (pl.col(c).last() - pl.col(c).first()).alias(f\"{c}_delta\"),\n            pl.corr(\"sequence_counter\", c).alias(f\"{c}_corr_time\"),\n            pl.col(c).diff().mean().alias(f\"{c}_diff_mean\"),\n            pl.col(c).diff().std().alias(f\"{c}_diff_std\"),\n            pl.col(c).skew().alias(f\"{c}_skew\"),\n            pl.col(c).kurtosis().alias(f\"{c}_kurt\"),\n            pl.col(c).diff().abs().gt(0).sum().alias(f\"{c}_n_changes\")\n        ]\n        \n        # Segment features from original\n        agg_exprs += [\n            pl.when(pl.col(\"sequence_counter\") < 0.1 * pl.max(\"sequence_counter\"))\n              .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg1_mean\"),\n            pl.when(pl.col(\"sequence_counter\") > 0.9 * pl.max(\"sequence_counter\"))\n              .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg3_mean\"),\n        ]\n        \n        # Additional quantiles from original\n        agg_exprs += [\n            pl.col(c).quantile(0.25, \"nearest\").alias(f\"{c}_t25\"),\n            pl.col(c).quantile(0.75, \"nearest\").alias(f\"{c}_t75\"),\n        ]\n    \n    # === NEW ENHANCED FEATURES ===\n    \n    # 1. BFRB-specific movement patterns (7 features per stat_col = 49 total)\n    for c in stat_cols:\n        agg_exprs += [\n            # Peak detection - count significant changes\n            pl.col(c).diff().abs().gt(pl.col(c).std() * 0.5).sum().alias(f\"{c}_peak_count\"),\n            \n            # Movement intensity - mean absolute deviation\n            (pl.col(c) - pl.col(c).mean()).abs().mean().alias(f\"{c}_mad\"),\n            \n            # Range normalization - signal range relative to std\n            ((pl.col(c).max() - pl.col(c).min()) / (pl.col(c).std() + 1e-8)).alias(f\"{c}_range_norm\"),\n            \n            # Acceleration analysis - second derivative\n            pl.col(c).diff().diff().abs().mean().alias(f\"{c}_accel_mean\"),\n            pl.col(c).diff().diff().std().alias(f\"{c}_accel_std\"),\n            \n            # Movement consistency - coefficient of variation\n            (pl.col(c).std() / (pl.col(c).mean().abs() + 1e-8)).alias(f\"{c}_cv\"),\n            \n            # Energy measure - sum of squared values\n            (pl.col(c) ** 2).sum().alias(f\"{c}_energy\"),\n        ]\n    \n    # 2. Multi-axis interaction features (15 features)\n    # Acceleration magnitude and patterns\n    agg_exprs += [\n        # 3D acceleration magnitude\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().mean().alias(\"acc_mag_mean\"),\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().std().alias(\"acc_mag_std\"),\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().max().alias(\"acc_mag_max\"),\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().min().alias(\"acc_mag_min\"),\n        \n        # 3D rotation magnitude\n        (pl.col(\"rot_x\")**2 + pl.col(\"rot_y\")**2 + pl.col(\"rot_z\")**2).sqrt().mean().alias(\"rot_mag_mean\"),\n        (pl.col(\"rot_x\")**2 + pl.col(\"rot_y\")**2 + pl.col(\"rot_z\")**2).sqrt().std().alias(\"rot_mag_std\"),\n        \n        # Axis dominance - which axis has most movement\n        (pl.col(\"acc_x\").abs().mean() / (pl.col(\"acc_x\").abs().mean() + pl.col(\"acc_y\").abs().mean() + pl.col(\"acc_z\").abs().mean() + 1e-8)).alias(\"acc_x_dominance\"),\n        (pl.col(\"acc_y\").abs().mean() / (pl.col(\"acc_x\").abs().mean() + pl.col(\"acc_y\").abs().mean() + pl.col(\"acc_z\").abs().mean() + 1e-8)).alias(\"acc_y_dominance\"),\n        (pl.col(\"acc_z\").abs().mean() / (pl.col(\"acc_x\").abs().mean() + pl.col(\"acc_y\").abs().mean() + pl.col(\"acc_z\").abs().mean() + 1e-8)).alias(\"acc_z_dominance\"),\n        \n        # Cross-axis correlations (key pairs only)\n        pl.corr(\"acc_x\", \"acc_y\").alias(\"acc_xy_corr\"),\n        pl.corr(\"acc_y\", \"acc_z\").alias(\"acc_yz_corr\"),\n        pl.corr(\"acc_x\", \"acc_z\").alias(\"acc_xz_corr\"),\n        \n        # Rotation-acceleration relationship\n        pl.corr(\"acc_x\", \"rot_x\").alias(\"acc_rot_x_corr\"),\n        pl.corr(\"acc_y\", \"rot_y\").alias(\"acc_rot_y_corr\"),\n        pl.corr(\"acc_z\", \"rot_z\").alias(\"acc_rot_z_corr\"),\n    ]\n    \n    # 3. Temporal pattern analysis (14 features)\n    agg_exprs += [\n        # Sequence characteristics\n        pl.max(\"sequence_counter\").alias(\"seq_length\"),\n        pl.col(\"sequence_counter\").count().alias(\"seq_count\"),\n        \n        # Early vs late behavior comparison for acceleration\n        (pl.when(pl.col(\"sequence_counter\") < 0.3 * pl.max(\"sequence_counter\"))\n         .then(pl.col(\"acc_x\")).otherwise(None).std() / \n         (pl.when(pl.col(\"sequence_counter\") > 0.7 * pl.max(\"sequence_counter\"))\n          .then(pl.col(\"acc_x\")).otherwise(None).std() + 1e-8)).alias(\"acc_x_early_late_ratio\"),\n        \n        (pl.when(pl.col(\"sequence_counter\") < 0.3 * pl.max(\"sequence_counter\"))\n         .then(pl.col(\"acc_y\")).otherwise(None).std() / \n         (pl.when(pl.col(\"sequence_counter\") > 0.7 * pl.max(\"sequence_counter\"))\n          .then(pl.col(\"acc_y\")).otherwise(None).std() + 1e-8)).alias(\"acc_y_early_late_ratio\"),\n        \n        (pl.when(pl.col(\"sequence_counter\") < 0.3 * pl.max(\"sequence_counter\"))\n         .then(pl.col(\"acc_z\")).otherwise(None).std() / \n         (pl.when(pl.col(\"sequence_counter\") > 0.7 * pl.max(\"sequence_counter\"))\n          .then(pl.col(\"acc_z\")).otherwise(None).std() + 1e-8)).alias(\"acc_z_early_late_ratio\"),\n        \n        # Movement intensity over time\n        (pl.col(\"acc_x\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"acc_x_intensity_rate\"),\n        (pl.col(\"acc_y\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"acc_y_intensity_rate\"),\n        (pl.col(\"acc_z\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"acc_z_intensity_rate\"),\n        \n        # Rotation intensity over time  \n        (pl.col(\"rot_x\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"rot_x_intensity_rate\"),\n        (pl.col(\"rot_y\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"rot_y_intensity_rate\"),\n        (pl.col(\"rot_z\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"rot_z_intensity_rate\"),\n        \n        # Total movement complexity\n        ((pl.col(\"acc_x\").diff().abs().sum() + pl.col(\"acc_y\").diff().abs().sum() + pl.col(\"acc_z\").diff().abs().sum()) / \n         (pl.max(\"sequence_counter\") + 1)).alias(\"total_acc_complexity\"),\n        \n        ((pl.col(\"rot_x\").diff().abs().sum() + pl.col(\"rot_y\").diff().abs().sum() + pl.col(\"rot_z\").diff().abs().sum()) / \n         (pl.max(\"sequence_counter\") + 1)).alias(\"total_rot_complexity\"),\n        \n        # Movement smoothness (inverse of jerk)\n        pl.col(\"acc_x\").diff().diff().abs().sum().alias(\"acc_x_jerk\"),\n        pl.col(\"acc_y\").diff().diff().abs().sum().alias(\"acc_y_jerk\"),\n    ]\n    \n    # === EXISTING PROCESSING (unchanged) ===\n    result = (\n        sequence.\n        group_by(pl.col([\"sequence_id\", \"subject\"]), maintain_order=True).\n        agg(agg_exprs).\n        select(pl.all().shrink_dtype()).\n        join(\n            demographics.select(pl.all().shrink_dtype()),\n            how=\"left\",\n            on=[\"subject\"],\n        ).\n        drop([\"subject\"], strict=False).\n        to_pandas()\n    )\n    \n    # Handle NaN values with more robust approach\n    result = result.fillna(0.0)\n    \n    # Remove any potential infinite values\n    result = result.replace([np.inf, -np.inf], 0.0)\n    \n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:38.905410Z","iopub.execute_input":"2025-06-18T04:47:38.905861Z","iopub.status.idle":"2025-06-18T04:47:39.094505Z","shell.execute_reply.started":"2025-06-18T04:47:38.905835Z","shell.execute_reply":"2025-06-18T04:47:39.093922Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Loading & Preprocessing","metadata":{}},{"cell_type":"code","source":"print(\"Loading data...\")\ntrain = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\ntraind = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\ntest = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\ntestd = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")\n\n# Select IMU-only columns (consistent with both notebooks)\nsel_cols = [c for c in test.collect_schema().names() if \"thm\" not in c and \"tof\" not in c]\n\nprint(\"Extracting enhanced features (~220 dimensions)...\")\nXtrain = make_ftre_unified_enhanced(train.select(pl.col(sel_cols)), traind)\nXtest = make_ftre_unified_enhanced(test.select(pl.col(sel_cols)), testd)\n\nprint(f\"Enhanced training features shape: {Xtrain.shape}\")\nprint(f\"Enhanced test features shape: {Xtest.shape}\")\n\n# Prepare targets\nYtrain = train.select(pl.col([\"sequence_id\", \"subject\", target])).unique().to_pandas()\nYtrain[target] = Ytrain[target].map(mapper).astype(np.int32)\nYtrain = Xtrain[[\"sequence_id\"]].merge(Ytrain, how=\"inner\", on=[\"sequence_id\"])\nytrain = Ytrain[target].values\n\nprint(f\"Target distribution:\")\nprint(pd.Series(ytrain).value_counts().sort_index())\n\n# Cross-validation setup (consistent with both notebooks)\ncv = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n\nygrp = np.zeros(len(Xtrain))\nfor fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, Ytrain[target], Ytrain[grouper])):\n    ygrp[dev_idx] = fold_nb\n\ncv = PredefinedSplit(ygrp)\nprint(f\"Cross-validation setup complete. Shapes: {Xtrain.shape}, {ytrain.shape}, {Xtest.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:39.095301Z","iopub.execute_input":"2025-06-18T04:47:39.095510Z","iopub.status.idle":"2025-06-18T04:47:53.446027Z","shell.execute_reply.started":"2025-06-18T04:47:39.095488Z","shell.execute_reply":"2025-06-18T04:47:53.445210Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TabM Classifier","metadata":{}},{"cell_type":"code","source":"class TabMClassifier:\n    \"\"\"TabM Classifier for CMI BFRB task - Enhanced with unified features\"\"\"\n    \n    def __init__(\n        self,\n        arch_type: str = 'tabm-mini',\n        backbone: dict = {'type': 'MLP', 'n_blocks': 3, 'd_block': 256, 'dropout': 0.15},\n        d_embedding: int = 64,\n        bin_count: int = 52,\n        k: int = 8,\n        learning_rate: float = 1e-3,\n        weight_decay: float = 1e-4,\n        max_epochs: int = 100,\n        patience: int = 20,\n        batch_size: int = 32,\n        device: Optional[str] = None,\n        random_state: int = 42,\n        verbose: bool = True\n    ):\n        self.arch_type = arch_type\n        self.backbone = backbone\n        self.d_embedding = d_embedding\n        self.bin_count = bin_count\n        self.k = k\n        self.learning_rate = learning_rate\n        self.weight_decay = weight_decay\n        self.max_epochs = max_epochs\n        self.patience = patience\n        self.batch_size = batch_size\n        self.device = torch.device(device if device else ('cuda' if torch.cuda.is_available() else 'cpu'))\n        self.random_state = random_state\n        self.verbose = verbose\n        self.n_classes = 18\n        \n    def fit(self, X: pd.DataFrame, y: np.array, eval_set: Tuple[pd.DataFrame, np.array]):\n        \"\"\"Fit TabM classifier with enhanced features\"\"\"\n        \n        # Preprocess data\n        X_cat_train, X_cont_train, cat_cardinalities, y_train = self._preprocess_data(X, y, training=True)\n        X_cat_val, X_cont_val, _, y_val = self._preprocess_data(eval_set[0], eval_set[1], training=False)\n        \n        if self.verbose:\n            print(f\"Training data shapes: X_cont: {X_cont_train.shape}, X_cat: {X_cat_train.shape}\")\n            print(f\"Validation data shapes: X_cont: {X_cont_val.shape}, X_cat: {X_cat_val.shape}\")\n        \n        # Create model\n        bins = None\n        if self.arch_type == 'tabm-mini' and X_cont_train.shape[1] > 0:\n            try:\n                bins = rtdl_num_embeddings.compute_bins(X_cont_train, n_bins=self.bin_count)\n            except Exception as e:\n                if self.verbose:\n                    print(f\"Warning: Could not compute bins, using None: {e}\")\n                bins = None\n        \n        self.model = Model(\n            n_num_features=X_cont_train.shape[1],\n            cat_cardinalities=cat_cardinalities,\n            n_classes=self.n_classes,\n            backbone=self.backbone,\n            bins=bins,\n            num_embeddings=(\n                None if bins is None else {\n                    'type': 'PiecewiseLinearEmbeddings',\n                    'd_embedding': self.d_embedding,\n                    'activation': False,\n                    'version': 'B',\n                }\n            ),\n            arch_type=self.arch_type,\n            k=self.k,\n        ).to(self.device)\n        \n        optimizer = torch.optim.AdamW(\n            make_parameter_groups(self.model), \n            lr=self.learning_rate, \n            weight_decay=self.weight_decay\n        )\n        \n        loss_fn = torch.nn.CrossEntropyLoss().to(self.device)\n        \n        # Training loop\n        best = {\n            'epoch': -1,\n            'eval_score': -np.inf,\n            'model_state_dict': None,\n        }\n        remaining_patience = self.patience\n        \n        for epoch in range(self.max_epochs):\n            # Training phase\n            self.model.train()\n            train_losses = []\n            \n            indices = torch.randperm(len(y_train), device=self.device).split(self.batch_size)\n            \n            for batch_idx in indices:\n                optimizer.zero_grad()\n                \n                try:\n                    if X_cat_train.shape[1] > 0:\n                        logits = self.model(X_cont_train[batch_idx], X_cat_train[batch_idx])\n                    else:\n                        logits = self.model(X_cont_train[batch_idx], None)\n                    \n                    logits_flat = logits.contiguous().reshape(-1, self.n_classes)\n                    y_repeated = y_train[batch_idx].repeat_interleave(self.k)\n                    \n                    loss = loss_fn(logits_flat, y_repeated)\n                    \n                except Exception as e:\n                    print(f\"Error in forward pass: {e}\")\n                    raise e\n                \n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)\n                optimizer.step()\n                \n                train_losses.append(loss.item())\n            \n            # Validation phase\n            self.model.eval()\n            val_preds = []\n            val_true = []\n            \n            with torch.no_grad():\n                for batch_idx in range(0, len(y_val), self.batch_size):\n                    end_idx = min(batch_idx + self.batch_size, len(y_val))\n                    \n                    if X_cat_val.shape[1] > 0:\n                        logits = self.model(\n                            X_cont_val[batch_idx:end_idx],\n                            X_cat_val[batch_idx:end_idx]\n                        )\n                    else:\n                        logits = self.model(\n                            X_cont_val[batch_idx:end_idx],\n                            None\n                        )\n                    \n                    logits = logits.contiguous()\n                    probs = torch.softmax(logits, dim=-1).mean(dim=1)\n                    preds = probs.argmax(dim=-1).cpu().numpy()\n                    \n                    val_preds.extend(preds)\n                    val_true.extend(y_val[batch_idx:end_idx].cpu().numpy())\n            \n            # Calculate CMI score\n            score, bscore, mscore = ScoreMetric(np.array(val_true), np.array(val_preds))\n            \n            if self.verbose:\n                print(f'Epoch {epoch} | Train Loss: {np.mean(train_losses):.4f} | '\n                      f'Val Score: {score:.4f} | Binary F1: {bscore:.4f} | Macro F1: {mscore:.4f}')\n            \n            # Check for improvement\n            if score > best['eval_score']:\n                best['epoch'] = epoch\n                best['eval_score'] = score\n                best['model_state_dict'] = self.model.state_dict()\n                remaining_patience = self.patience\n                \n                if self.verbose:\n                    print('🌸 New best epoch! 🌸')\n            else:\n                remaining_patience -= 1\n            \n            # Early stopping\n            if remaining_patience == 0:\n                if self.verbose:\n                    print(f\"Early stopping at epoch {epoch}\")\n                break\n        \n        # Restore best model\n        self.model.load_state_dict(best['model_state_dict'])\n        if self.verbose:\n            print(f\"Best model restored from epoch {best['epoch']} with score {best['eval_score']:.4f}\")\n        \n    def predict(self, X: pd.DataFrame, batch_size: Optional[int] = 1024) -> np.ndarray:\n        \"\"\"Predict classes\"\"\"\n        X_cat, X_cont, _, _ = self._preprocess_data(X, y=None, training=False)\n        \n        self.model.eval()\n        predictions = []\n        \n        with torch.no_grad():\n            for batch_idx in range(0, len(X), batch_size):\n                end_idx = min(batch_idx + batch_size, len(X))\n                \n                if X_cat.shape[1] > 0:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        X_cat[batch_idx:end_idx]\n                    )\n                else:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        None\n                    )\n                \n                logits = logits.contiguous()\n                probs = torch.softmax(logits, dim=-1).mean(dim=1)\n                preds = probs.argmax(dim=-1).cpu().numpy()\n                \n                predictions.extend(preds)\n        \n        return np.array(predictions)\n    \n    def predict_proba(self, X: pd.DataFrame, batch_size: Optional[int] = 1024) -> np.ndarray:\n        \"\"\"Predict class probabilities\"\"\"\n        X_cat, X_cont, _, _ = self._preprocess_data(X, y=None, training=False)\n        \n        self.model.eval()\n        probabilities = []\n        \n        with torch.no_grad():\n            for batch_idx in range(0, len(X), batch_size):\n                end_idx = min(batch_idx + batch_size, len(X))\n                \n                if X_cat.shape[1] > 0:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        X_cat[batch_idx:end_idx]\n                    )\n                else:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        None\n                    )\n                \n                logits = logits.contiguous()\n                probs = torch.softmax(logits, dim=-1).mean(dim=1).cpu().numpy()\n                probabilities.append(probs)\n        \n        return np.vstack(probabilities)\n    \n    def _preprocess_data(self, X: pd.DataFrame, y: Optional[np.array], training: bool):\n        \"\"\"Preprocess data for TabM\"\"\"\n        \n        # Handle non-constant columns\n        if training:\n            self._non_constant_columns = X.columns[X.nunique() > 1]\n        \n        X = X[self._non_constant_columns].copy()\n        \n        # Separate categorical and continuous features\n        categorical_features = ['adult_child', 'sex', 'handedness'] if 'adult_child' in X.columns else []\n        \n        if categorical_features:\n            X_cat = X[categorical_features].to_numpy()\n        else:\n            X_cat = np.empty((len(X), 0))\n            \n        X_cont = X.drop(columns=categorical_features, errors='ignore').to_numpy().astype(np.float32)\n        \n        # Handle categorical features\n        cat_cardinalities = []\n        if X_cat.shape[1] > 0:\n            if training:\n                self._categorical_encoders = []\n                for i, col in enumerate(categorical_features):\n                    encoder = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)\n                    self._categorical_encoders.append(encoder)\n                    cat_cardinalities.append(X[col].nunique() + 1)\n            \n            X_cat_encoded = []\n            for i, encoder in enumerate(self._categorical_encoders):\n                if training:\n                    encoded = encoder.fit_transform(X_cat[:, i:i+1])\n                else:\n                    encoded = encoder.transform(X_cat[:, i:i+1])\n                X_cat_encoded.append(encoded)\n            \n            if X_cat_encoded:\n                X_cat = np.concatenate(X_cat_encoded, axis=1).astype(np.int64)\n            else:\n                X_cat = np.empty((len(X), 0), dtype=np.int64)\n        else:\n            X_cat = np.empty((len(X), 0), dtype=np.int64)\n        \n        # Handle continuous features\n        if training:\n            if X_cont.shape[1] > 0:\n                noise = np.random.default_rng(0).normal(0.0, 1e-5, X_cont.shape).astype(X_cont.dtype)\n                self._cont_feature_preprocessor = MinMaxScaler().fit(X_cont + noise)\n            else:\n                self._cont_feature_preprocessor = MinMaxScaler()\n        \n        if X_cont.shape[1] > 0:\n            X_cont = self._cont_feature_preprocessor.transform(X_cont)\n        \n        # Convert to tensors\n        X_cat = torch.tensor(X_cat, dtype=torch.long, device=self.device)\n        X_cont = torch.tensor(X_cont, dtype=torch.float32, device=self.device)\n        \n        if y is not None:\n            y = torch.tensor(y, dtype=torch.long, device=self.device)\n        \n        return X_cat, X_cont, cat_cardinalities, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:53.446946Z","iopub.execute_input":"2025-06-18T04:47:53.447199Z","iopub.status.idle":"2025-06-18T04:47:53.473452Z","shell.execute_reply.started":"2025-06-18T04:47:53.447181Z","shell.execute_reply":"2025-06-18T04:47:53.472934Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TabM Training Pipeline","metadata":{}},{"cell_type":"code","source":"def train_tabm_models():\n    \"\"\"Train 5 TabM models with enhanced features\"\"\"\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"TRAINING TABM MODELS (5-FOLD CV) - ENHANCED FEATURES\")\n    print(\"=\"*70)\n    \n    tabm_oof_preds = []\n    tabm_fitted_models = []\n    drop_cols = [\"sequence_id\", target, grouper]\n    \n    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain), start=1):\n        print(f\"\\n{'='*50}\")\n        print(f\"Training TabM Fold {fold_nb}/5\")\n        print(f\"{'='*50}\")\n        \n        Xtr = Xtrain.iloc[train_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        Xdev = Xtrain.iloc[dev_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        ytr = ytrain[train_idx]\n        ydev = ytrain[dev_idx]\n        \n        print(f\"TabM Fold {fold_nb} - Train: {len(Xtr)}, Val: {len(Xdev)}\")\n        \n        # Initialize and train model\n        model = TabMClassifier(\n            arch_type='tabm-mini',\n            backbone={'type': 'MLP', 'n_blocks': 3, 'd_block': 256, 'dropout': 0.15},\n            d_embedding=64,\n            k=8,\n            learning_rate=1e-3,\n            weight_decay=1e-4,\n            max_epochs=30,\n            patience=8,\n            batch_size=16,\n            verbose=True\n        )\n        \n        try:\n            model.fit(Xtr, ytr, eval_set=(Xdev, ydev))\n            tabm_fitted_models.append(model)\n            \n            # Get predictions\n            dev_preds = model.predict(Xdev)\n            tabm_oof_preds.append(pd.DataFrame({'pred': dev_preds}, index=dev_idx))\n            \n            # Calculate score\n            score, bscore, mscore = ScoreMetric(ydev, dev_preds)\n            print(f\"\\nTabM Fold {fold_nb} Results:\")\n            print(f\"Overall Score: {score:.6f}\")\n            print(f\"Binary F1: {bscore:.6f}\")\n            print(f\"Macro F1: {mscore:.6f}\")\n            \n        except Exception as e:\n            print(f\"Error in TabM fold {fold_nb}: {e}\")\n            print(\"Skipping this fold...\")\n            continue\n    \n    if not tabm_fitted_models:\n        raise RuntimeError(\"No TabM models were successfully trained!\")\n    \n    # Combine OOF predictions\n    tabm_oof_preds = pd.concat(tabm_oof_preds, axis=0).sort_index()\n    \n    # Calculate overall score\n    overall_score, overall_bscore, overall_mscore = ScoreMetric(ytrain, tabm_oof_preds['pred'].values)\n    print(f\"\\n{'='*50}\")\n    print(f\"TABM OVERALL RESULTS - ENHANCED\")\n    print(f\"{'='*50}\")\n    print(f\"Overall Score: {overall_score:.6f}\")\n    print(f\"Binary F1: {overall_bscore:.6f}\")\n    print(f\"Macro F1: {overall_mscore:.6f}\")\n    \n    return tabm_fitted_models, tabm_oof_preds\n\n# Execute TabM training\ntabm_models, tabm_oof = train_tabm_models()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:47:53.475785Z","iopub.execute_input":"2025-06-18T04:47:53.476028Z","iopub.status.idle":"2025-06-18T04:51:43.399019Z","shell.execute_reply.started":"2025-06-18T04:47:53.476012Z","shell.execute_reply":"2025-06-18T04:51:43.398351Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CatBoost Training Pipeline","metadata":{}},{"cell_type":"code","source":"def train_catboost_models():\n    \"\"\"Train 5 CatBoost models with enhanced features\"\"\"\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"TRAINING CATBOOST MODELS (5-FOLD CV) - ENHANCED FEATURES\")\n    print(\"=\"*70)\n    \n    catboost_model = CBC(\n        loss_function=\"MultiClass\",\n        task_type=\"CPU\" if torch.cuda.is_available() == False else \"GPU\",\n        iterations=10000,\n        max_depth=8,\n        learning_rate=0.03,\n        colsample_bylevel=0.50 if torch.cuda.is_available() == False else None,\n        random_state=42,\n        early_stopping_rounds=200,\n        l2_leaf_reg=0.40,\n        verbose=0,\n    )\n    \n    catboost_oof_preds = []\n    catboost_fitted_models = []\n    drop_cols = [\"sequence_id\", target, grouper]\n    \n    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain), start=1):\n        print(f\"\\nTraining CatBoost Fold {fold_nb}/5\")\n        \n        Xtr = Xtrain.iloc[train_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        Xdev = Xtrain.iloc[dev_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        ytr = ytrain[train_idx]\n        ydev = ytrain[dev_idx]\n        \n        model = clone(catboost_model)\n        model.fit(\n            Xtr, ytr, eval_set=[(Xdev, ydev)], verbose=0\n        )\n        dev_preds = pd.DataFrame(model.predict_proba(Xdev), index=dev_idx)\n        catboost_oof_preds.append(dev_preds)\n        catboost_fitted_models.append(model)\n        \n        score, bscore, mscore = ScoreMetric(\n            ydev.flatten(), \n            dev_preds.idxmax(axis=1).values.flatten()\n        )\n        print(f\"CatBoost Fold {fold_nb} - Score: {score:.6f} | Binary F1: {bscore:.6f} | Macro F1: {mscore:.6f}\")\n    \n    catboost_oof_preds = pd.concat(catboost_oof_preds, axis=0).sort_index(ascending=True)\n    \n    score, bscore, mscore = ScoreMetric(\n        ytrain.flatten(), \n        catboost_oof_preds.idxmax(axis=1).values.flatten()\n    )\n    print(f\"\\n{'='*50}\")\n    print(f\"CATBOOST OVERALL RESULTS - ENHANCED\")\n    print(f\"{'='*50}\")\n    print(f\"Overall Score: {score:.6f}\")\n    print(f\"Binary F1: {bscore:.6f}\")\n    print(f\"Macro F1: {mscore:.6f}\")\n    \n    collect()\n    return catboost_fitted_models, catboost_oof_preds\n\n# Execute CatBoost training\ncatboost_models, catboost_oof = train_catboost_models()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:51:43.399832Z","iopub.execute_input":"2025-06-18T04:51:43.400377Z","iopub.status.idle":"2025-06-18T04:58:21.330804Z","shell.execute_reply.started":"2025-06-18T04:51:43.400345Z","shell.execute_reply":"2025-06-18T04:58:21.330000Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble & Validation","metadata":{}},{"cell_type":"code","source":"def validate_ensemble_strategies():\n    \"\"\"Test different ensemble strategies with proper probability handling\"\"\"\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"ENSEMBLE STRATEGY VALIDATION - ENHANCED FEATURES\")\n    print(\"=\"*70)\n    \n    # First, let's get TabM probabilities properly\n    print(\"Getting TabM probabilities from trained models...\")\n    tabm_probs_oof = []\n    drop_cols = [\"sequence_id\", target, grouper]\n    \n    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain)):\n        Xdev = Xtrain.iloc[dev_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        try:\n            # Get probabilities from the trained TabM model\n            dev_probs = tabm_models[fold_nb].predict_proba(Xdev)\n            tabm_probs_oof.append(pd.DataFrame(dev_probs, index=dev_idx))\n        except Exception as e:\n            print(f\"Warning: Could not get probabilities from TabM model {fold_nb}, using predictions: {e}\")\n            # Fallback: use soft probabilities instead of hard one-hot\n            dev_preds = tabm_models[fold_nb].predict(Xdev) \n            soft_probs = np.zeros((len(dev_preds), 18))\n            soft_probs[np.arange(len(dev_preds)), dev_preds] = 0.8  # 80% confidence\n            # Distribute remaining 20% equally among other classes\n            for i in range(len(dev_preds)):\n                remaining = 0.2 / 17  # Distribute among 17 other classes\n                soft_probs[i, :] += remaining\n                soft_probs[i, dev_preds[i]] = 0.8  # Reset main prediction to 80%\n            tabm_probs_oof.append(pd.DataFrame(soft_probs, index=dev_idx))\n    \n    tabm_probs_oof = pd.concat(tabm_probs_oof, axis=0).sort_index()\n    \n    print(\"TabM probabilities shape:\", tabm_probs_oof.shape)\n    print(\"CatBoost probabilities shape:\", catboost_oof.shape)\n    \n    # Strategy 1: Equal weight (0.5:0.5) - NOW WITH REAL PROBABILITIES\n    def equal_weight_ensemble(tabm_probs, catboost_probs):\n        ensemble_probs = 0.5 * tabm_probs + 0.5 * catboost_probs\n        return np.argmax(ensemble_probs, axis=1)\n    \n    # Strategy 2: Performance-weighted ensemble\n    def weighted_ensemble(tabm_probs, catboost_probs, tabm_weight=0.7):\n        ensemble_probs = tabm_weight * tabm_probs + (1-tabm_weight) * catboost_probs\n        return np.argmax(ensemble_probs, axis=1)\n    \n    # Strategy 3: TabM only (from probabilities)\n    def tabm_only(tabm_probs, catboost_probs):\n        return np.argmax(tabm_probs, axis=1)\n    \n    # Strategy 4: CatBoost only\n    def catboost_only(tabm_probs, catboost_probs):\n        return np.argmax(catboost_probs, axis=1)\n    \n    strategies = {\n        \"TabM Only\": tabm_only,\n        \"CatBoost Only\": catboost_only,\n        \"Equal Weight (0.5:0.5)\": equal_weight_ensemble,\n        \"Weighted (0.3:0.7)\": lambda t, c: weighted_ensemble(t, c, 0.3),\n        \"Weighted (0.2:0.8)\": lambda t, c: weighted_ensemble(t, c, 0.2),\n        \"Weighted (0.4:0.6)\": lambda t, c: weighted_ensemble(t, c, 0.4),\n    }\n    \n    results = {}\n    \n    for strategy_name, strategy_func in strategies.items():\n        ensemble_preds = strategy_func(tabm_probs_oof.values, catboost_oof.values)\n        score, bscore, mscore = ScoreMetric(ytrain, ensemble_preds)\n        results[strategy_name] = {\n            'score': score,\n            'binary_f1': bscore,\n            'macro_f1': mscore\n        }\n        print(f\"{strategy_name:25} - Score: {score:.6f} | Binary F1: {bscore:.6f} | Macro F1: {mscore:.6f}\")\n    \n    # Find best strategy\n    best_strategy = max(results.keys(), key=lambda k: results[k]['score'])\n    best_score = results[best_strategy]['score']\n    \n    print(f\"\\n🏆 Best Strategy: {best_strategy} with score {best_score:.6f}\")\n    \n    return best_strategy, strategies[best_strategy], tabm_probs_oof\n\n# Validate ensemble strategies with proper probabilities\nbest_strategy_name, best_strategy_func, tabm_probs_oof = validate_ensemble_strategies()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:58:21.331749Z","iopub.execute_input":"2025-06-18T04:58:21.332054Z","iopub.status.idle":"2025-06-18T04:58:21.640780Z","shell.execute_reply.started":"2025-06-18T04:58:21.332036Z","shell.execute_reply":"2025-06-18T04:58:21.639755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Final Ensemble Prediction Function","metadata":{}},{"cell_type":"code","source":"def create_ensemble_prediction_function(tabm_models, catboost_models, strategy_func):\n    \"\"\"Create unified prediction function with real probability ensemble\"\"\"\n    \n    def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n        \"\"\"Ensemble prediction function with real probabilities\"\"\"\n        try:\n            # Extract enhanced features\n            drop_cols = [\"sequence_id\", target, grouper]\n            Xtest = make_ftre_unified_enhanced(sequence, demographics).drop(drop_cols, axis=1, errors=\"ignore\")\n            \n            # Get TabM probabilities (average across 5 models)\n            tabm_probs_list = []\n            for model in tabm_models:\n                try:\n                    pred_probs = model.predict_proba(Xtest)\n                    tabm_probs_list.append(pred_probs)\n                except Exception as e:\n                    print(f\"Warning: TabM predict_proba failed: {e}, using soft prediction\")\n                    # Soft prediction fallback\n                    pred = model.predict(Xtest)\n                    soft_probs = np.zeros((len(Xtest), 18))\n                    soft_probs[np.arange(len(Xtest)), pred] = 0.8\n                    remaining = 0.2 / 17\n                    soft_probs += remaining\n                    soft_probs[np.arange(len(Xtest)), pred] = 0.8\n                    tabm_probs_list.append(soft_probs)\n            \n            # Average TabM probabilities\n            avg_tabm_probs = np.mean(tabm_probs_list, axis=0)  # Shape: (1, 18)\n            \n            # Get CatBoost probabilities (average across 5 models)\n            catboost_probs_list = []\n            for model in catboost_models:\n                pred_probs = model.predict_proba(Xtest)\n                catboost_probs_list.append(pred_probs)\n            \n            # Average CatBoost probabilities\n            avg_catboost_probs = np.mean(catboost_probs_list, axis=0)  # Shape: (1, 18)\n            \n            # Apply ensemble strategy with REAL probabilities\n            final_pred = strategy_func(avg_tabm_probs, avg_catboost_probs)[0]\n            \n            # Individual predictions for logging\n            tabm_pred = np.argmax(avg_tabm_probs, axis=1)[0]\n            catboost_pred = np.argmax(avg_catboost_probs, axis=1)[0]\n            \n            # Convert back to gesture name\n            gesture_name = {v: k for k, v in mapper.items()}[final_pred]\n            \n            print(f\"TabM pred: {tabm_pred} (conf: {avg_tabm_probs[0, tabm_pred]:.3f})\")\n            print(f\"CatBoost pred: {catboost_pred} (conf: {avg_catboost_probs[0, catboost_pred]:.3f})\")\n            print(f\"Ensemble pred: {final_pred} -> {gesture_name}\")\n            \n            return gesture_name\n            \n        except Exception as e:\n            print(f\"Error in ensemble prediction: {e}\")\n            import traceback\n            traceback.print_exc()\n            return \"Forehead - scratch\"\n    \n    return predict\n\n# Create final prediction function with guaranteed ensemble\nensemble_predict_fn = create_ensemble_prediction_function(tabm_models, catboost_models, best_strategy_func)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:58:21.641778Z","iopub.execute_input":"2025-06-18T04:58:21.642185Z","iopub.status.idle":"2025-06-18T04:58:21.651781Z","shell.execute_reply.started":"2025-06-18T04:58:21.642157Z","shell.execute_reply":"2025-06-18T04:58:21.650988Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Kaggle Submission Setup","metadata":{}},{"cell_type":"code","source":"import kaggle_evaluation.cmi_inference_server\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"SETTING UP KAGGLE SUBMISSION - ENHANCED FEATURES\")\nprint(\"=\"*70)\nprint(f\"Total TabM models: {len(tabm_models)}\")\nprint(f\"Total CatBoost models: {len(catboost_models)}\")\nprint(f\"Total ensemble models: {len(tabm_models) + len(catboost_models)}\")\nprint(f\"Best ensemble strategy: {best_strategy_name}\")\nprint(f\"Feature dimensions: ~220 (enhanced)\")\n\n# Setup inference server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(ensemble_predict_fn)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    print(\"Running in competition mode...\")\n    inference_server.serve()\nelse:\n    print(\"Running in local test mode...\")\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T04:58:21.652566Z","iopub.execute_input":"2025-06-18T04:58:21.653050Z","iopub.status.idle":"2025-06-18T04:58:23.040040Z","shell.execute_reply.started":"2025-06-18T04:58:21.653030Z","shell.execute_reply":"2025-06-18T04:58:23.039265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}