{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":10097128,"sourceType":"datasetVersion","datasetId":6226248},{"sourceId":242895646,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# References\n- https://www.kaggle.com/code/ravi20076/cmi2025-public-baseline-v3?scriptVersionId=244089160\n- https://www.kaggle.com/code/yekenot/cibmtr-tabm-nn/notebook\n\n## Enhanced Feature Engineering Results\n\n### Performance Improvement\n**Cross-Validation Results from this notebook V1:**\n- **TabM**: 0.686638 → **0.716331** (+0.030)\n- **CatBoost**: 0.691123 → **0.717714** (+0.026)\n- **Public LB**: 0.70 → **0.72** (+0.02)\n\n### Added Features (~65 new features, 155→220 dimensions)\n\n#### 1. BFRB-Specific Movement Patterns (49 features)\n- **Peak detection**: Count significant changes for repetitive behavior detection\n- **Movement intensity**: Mean absolute deviation to measure action force\n- **Range normalization**: Signal range relative to standard deviation\n- **Acceleration analysis**: Second derivative for sudden movement detection\n- **Movement consistency**: Coefficient of variation for pattern regularity\n- **Energy measure**: Sum of squared values for motion intensity\n\n#### 2. Multi-Axis Interaction Features (15 features)\n- **3D magnitude**: Combined acceleration/rotation vector magnitudes\n- **Axis dominance**: Which axis contributes most to movement\n- **Cross-correlations**: Inter-axis relationships (acc_xy, acc_yz, etc.)\n- **Rotation-acceleration coupling**: Correlation between rotation and linear motion\n\n#### 3. Temporal Pattern Analysis (14 features)\n- **Sequence characteristics**: Length and count metrics\n- **Early vs late behavior**: Variance ratio between sequence phases\n- **Movement intensity rates**: Change density over time\n- **Motion complexity**: Total variation normalized by sequence length\n- **Smoothness metrics**: Jerk analysis for movement quality assessment\n\nThese enhancements specifically target **repetitive and directional patterns** characteristic of BFRB behaviors, leading to improved discrimination between target and non-target gestures.","metadata":{}},{"cell_type":"markdown","source":"### If you find this notebook useful, please upvote this and the original notebooks!","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --no-deps --find-links=/kaggle/input/tabm-tabular-dl-library rtdl_num_embeddings==0.0.11\n!pip install --no-index --no-deps --find-links=/kaggle/input/tabm-tabular-dl-library tabm==0.0.1.dev0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:24:50.601212Z","iopub.execute_input":"2025-08-18T14:24:50.601540Z","iopub.status.idle":"2025-08-18T14:24:54.148046Z","shell.execute_reply.started":"2025-08-18T14:24:50.601504Z","shell.execute_reply":"2025-08-18T14:24:54.146555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#xung đột môi trường: \n\n#!pip install -q polars==1.29.0      --no-index --find-links=/kaggle/input/cmi2025-public-imports-v2/packages\n#!pip install -q lightgbm==4.6.0     --no-index --find-links=/kaggle/input/cmi2025-public-imports-v2/packages\n#!pip install -q scikit-learn==1.6.1 --no-index --find-links=/kaggle/input/cmi2025-public-imports-v2/packages","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:24:54.150455Z","iopub.execute_input":"2025-08-18T14:24:54.150777Z","iopub.status.idle":"2025-08-18T14:24:54.155834Z","shell.execute_reply.started":"2025-08-18T14:24:54.150745Z","shell.execute_reply":"2025-08-18T14:24:54.154869Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import & Setup","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nimport joblib\nfrom tqdm.notebook import tqdm\nfrom gc import collect\nimport warnings\nimport math\nimport random\nfrom typing import Optional, Tuple\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings\nfrom matplotlib.ticker import MaxNLocator\nfrom mpl_toolkits.mplot3d import Axes3D\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nfrom sklearn.metrics import accuracy_score, f1_score\nimport joblib\nfrom scipy.spatial.transform import Rotation as R\n\nfrom sklearn.base import clone, BaseEstimator\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler, OrdinalEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold, PredefinedSplit\nfrom sklearn.metrics import f1_score\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.ensemble import BaggingClassifier as BC\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings\nfrom matplotlib.ticker import MaxNLocator\nfrom mpl_toolkits.mplot3d import Axes3D\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nfrom sklearn.metrics import accuracy_score, f1_score\nimport joblib\nfrom scipy.spatial.transform import Rotation as R\n\n# LightGBM and CatBoost\nfrom lightgbm import LGBMClassifier as LGBMC, log_evaluation, early_stopping\nfrom catboost import CatBoostClassifier as CBC\n\n# TabM imports\nimport sys\nsys.path.append('/kaggle/input/tabm-tabular-dl-library')\nfrom tabm_reference import Model, make_parameter_groups\nimport rtdl_num_embeddings\n\nwarnings.filterwarnings('ignore')\n\ndef seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.benchmark = True\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:24:54.156856Z","iopub.execute_input":"2025-08-18T14:24:54.157700Z","iopub.status.idle":"2025-08-18T14:25:08.332504Z","shell.execute_reply.started":"2025-08-18T14:24:54.157665Z","shell.execute_reply":"2025-08-18T14:25:08.331660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tải tập dữ liệu\n#Dữ liệu cảm biến cho tập huấn luyện (Train Sensor Data): \ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\n#Thông tin nhân khẩu học của đối tượng trong tập huấn luyện (giới tính, tuổi, ...): \ntrain_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\n\n#Dữ liệu cảm biến cho tập kiểm tra:\ntest_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\n\n# Thông tin nhân khẩu học của đối tượng trong tập kiểm tra:\ntest_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:25:08.333899Z","iopub.execute_input":"2025-08-18T14:25:08.334666Z","iopub.status.idle":"2025-08-18T14:25:47.641841Z","shell.execute_reply.started":"2025-08-18T14:25:08.334628Z","shell.execute_reply":"2025-08-18T14:25:47.640716Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TỔNG QUAN DỮ LIỆU","metadata":{}},{"cell_type":"markdown","source":"## Khám phá dữ liệu","metadata":{}},{"cell_type":"markdown","source":"### Tóm tắt thống kê và tính năng","metadata":{}},{"cell_type":"code","source":"datasets = {\n    \"Train Data\": train_df,\n    \"Train Demographics\": train_dem_df,\n    \"Test Data\": test_df,\n    \"Test Demographics\": test_dem_df,\n}\n\nfor name, df in datasets.items():\n    print(f\"Missing values in {name}:\")\n    missing = df.isnull().sum()\n    missing = missing[missing > 0]\n    if not missing.empty:\n        print(missing)\n    else:\n        print(\"  ✅ No missing values.\")\n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:25:47.644575Z","iopub.execute_input":"2025-08-18T14:25:47.644914Z","iopub.status.idle":"2025-08-18T14:25:48.538718Z","shell.execute_reply.started":"2025-08-18T14:25:47.644890Z","shell.execute_reply":"2025-08-18T14:25:48.537728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"###  kiểm tra giá trị trùng lặp","metadata":{}},{"cell_type":"code","source":"#Kiểm tra trùng lặp (Duplicate Rows): .duplicated(): Trả về Series boolean, dòng nào trùng lặp sẽ là True.\n#                                     .sum(): Đếm tổng số dòng trùng lặp.\n#Kiểm tra xem dữ liệu có bản ghi nào bị trùng không — quan trọng cho việc làm sạch dữ liệu.\n\n# Đếm các hàng trùng lặp trong train_df\ntrain_duplicates = train_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong  test_df\ntest_duplicates = test_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong train_dem_df (optional)\ntrain_dem_duplicates = train_dem_df.duplicated().sum()\n# Đếm các hàng trùng lặp trong test_dem_df (optional)\ntest_dem_duplicates = test_dem_df.duplicated().sum()\n\n# In số lượng dòng trùng:\nprint(f\"Number of duplicate rows in train_df: {train_duplicates}\")\nprint(f\"Number of duplicate rows in test_df: {test_duplicates}\")\nprint(f\"Number of duplicate rows in train_dem_df: {train_dem_duplicates}\")\nprint(f\"Number of duplicate rows in test_dem_df: {test_dem_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:25:48.539672Z","iopub.execute_input":"2025-08-18T14:25:48.539945Z","iopub.status.idle":"2025-08-18T14:25:53.962228Z","shell.execute_reply.started":"2025-08-18T14:25:48.539925Z","shell.execute_reply":"2025-08-18T14:25:53.960982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"markdown","source":"### Tỷ lệ phần trăm Null","metadata":{}},{"cell_type":"markdown","source":"UTILS/HELPERS","metadata":{}},{"cell_type":"code","source":"def null_percent(df):\n    per=((df.isnull().sum()/len(df))*100).round(2)\n    return per[per>0]\n\nprint(\"Nan Values in Train data\")\nprint(null_percent(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:25:53.963193Z","iopub.execute_input":"2025-08-18T14:25:53.963445Z","iopub.status.idle":"2025-08-18T14:25:54.823844Z","shell.execute_reply.started":"2025-08-18T14:25:53.963424Z","shell.execute_reply":"2025-08-18T14:25:54.822889Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Quan sát hoặc xu hướng ban đầu","metadata":{}},{"cell_type":"code","source":"# --- 2.4 Hợp nhất dữ liệu nhân khẩu học và cảm biến ---\nmerged_train = pd.merge(train_df, train_dem_df, on=\"subject\", how=\"left\")\nmerged_test = pd.merge(test_df, test_dem_df, on=\"subject\", how=\"left\")\n\nprint(\"Merged Train Shape:\", merged_train.shape)\nprint(\"Merged Test Shape:\", merged_test.shape)\ndisplay(merged_train.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:25:54.824786Z","iopub.execute_input":"2025-08-18T14:25:54.825055Z","iopub.status.idle":"2025-08-18T14:25:55.762206Z","shell.execute_reply.started":"2025-08-18T14:25:54.825035Z","shell.execute_reply":"2025-08-18T14:25:55.761345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Thống kê mô tả toàn bộ merged_train ---\nmerged_train.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:25:55.763193Z","iopub.execute_input":"2025-08-18T14:25:55.763481Z","iopub.status.idle":"2025-08-18T14:26:07.167872Z","shell.execute_reply.started":"2025-08-18T14:25:55.763447Z","shell.execute_reply":"2025-08-18T14:26:07.166405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra thiếu giá trị & thống kê cảm biến: \n#Danh sách các cột không thuộc cảm biến (có thể là ID, thông tin khác). \n\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nsensor_cols = [col for col in train_df.columns if not col.startswith(excluded_prefixes)]\n\n# Sensor Data Summary for TRAIN\n#isnull().sum(): Đếm số giá trị bị thiếu.\nmissing_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    '[TRAIN] Missing Count': train_df[sensor_cols].isnull().sum().values,\n    '[TRAIN] Missing %': (train_df[sensor_cols].isnull().sum().values / len(train_df)) * 100\n})\n\n#nunique(): Đếm số lượng giá trị duy nhất.\nunique_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Unique Values [TRAIN]': train_df[sensor_cols].nunique().values\n})\n\n#dtypes: Lấy kiểu dữ liệu của từng cột.\ndtypes_sensor = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Data Type': train_df[sensor_cols].dtypes.values\n})\n\n# Merge all summaries (NO test set)\n#merge: Gộp các bảng thống kê thành bảng duy nhất theo Feature.\nsensor_summary = missing_sensor_train \\\n    .merge(unique_sensor_train, on='Feature', how='left') \\\n    .merge(dtypes_sensor, on='Feature', how='left')\n\n# Display styled DataFrame (mask NaNs just for styling)\n#fillna(0): Điền giá trị thiếu bằng 0 (cho đẹp mắt khi hiển thị).\n#.style.background_gradient: Tô màu nền theo giá trị giúp dễ nhìn.\nstyled_df = sensor_summary.fillna(0)\nstyled_df.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:07.169515Z","iopub.execute_input":"2025-08-18T14:26:07.169930Z","iopub.status.idle":"2025-08-18T14:26:08.485518Z","shell.execute_reply.started":"2025-08-18T14:26:07.169885Z","shell.execute_reply":"2025-08-18T14:26:08.484628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Thống kê tương tự cho nhân khẩu học: Tương tự như phần thống kê cảm biến nhưng áp dụng cho dữ liệu nhân khẩu học.\n\n# Cột nhân khẩu học (không loại trừ)\ndem_cols = train_dem_df.columns\n\n# Giá trị bị thiếu trong dữ liệu nhân khẩu học của train \nmissing_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    '[TRAIN DEMO] Missing Count': train_dem_df[dem_cols].isnull().sum().values,\n    '[TRAIN DEMO] Missing %': (train_dem_df[dem_cols].isnull().sum().values / len(train_dem_df)) * 100\n})\n\n# Giá trị duy nhất được tính trong dữ liệu nhân khẩu học của train \nunique_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    'Unique Values [TRAIN DEMO]': train_dem_df[dem_cols].nunique().values\n})\n\n# Data types\ndtypes_demo = pd.DataFrame({\n    'Feature': dem_cols,\n    'Data Type': train_dem_df[dem_cols].dtypes.values\n})\n\n# Tóm tắt hợp nhất (chỉ dành cho đào tạo)\ndemo_summary = (\n    missing_demo_train\n    .merge(unique_demo_train, on='Feature', how='left')\n    .merge(dtypes_demo, on='Feature', how='left')\n)\n\n# Hiển thị tóm tắt theo phong cách\ndemo_summary.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:08.486644Z","iopub.execute_input":"2025-08-18T14:26:08.486994Z","iopub.status.idle":"2025-08-18T14:26:08.520568Z","shell.execute_reply.started":"2025-08-18T14:26:08.486955Z","shell.execute_reply":"2025-08-18T14:26:08.519649Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sensor-Based Features (Tính năng dựa trên cảm biến)","metadata":{}},{"cell_type":"markdown","source":"NOTE:\nacc_: Gia tốc kế.\nrot_: Cảm biến quay.\nthm_: Nhiệt độ, độ ẩm.\ntof_: Khoảng cách.","metadata":{}},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"## Numerical Feature Distribution Analysis (Phân tích phân phối tính năng số) (Phân tích đơn biến)","metadata":{}},{"cell_type":"markdown","source":"### Sensor-Based Features (Tính năng dựa trên cảm biến)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# 1) Sao chép đào tạo và kiểm tra để chúng ta không sửa đổi DataFrame gốc\ntrain_temp = train_df.copy()\ntest_temp  = test_df.copy()\n\n# 2) GIA TỐC KẾ: tính toán độ lớn tại mỗi dấu thời gian\ntrain_temp['acc_mag'] = np.sqrt(\n    train_temp['acc_x']**2 + train_temp['acc_y']**2 + train_temp['acc_z']**2\n)\ntest_temp['acc_mag'] = np.sqrt(\n    test_temp['acc_x']**2 + test_temp['acc_y']**2 + test_temp['acc_z']**2\n)\n\n# 3) ROTATION: tính toán “góc quay” từ thành phần quaternion w\n# (Lưu ý: rot_w nằm trong [-1,1], do đó arccos hợp lệ. Chúng tôi bỏ qua NaN nếu có.)\ntrain_temp['rot_angle'] = 2 * np.arccos(train_temp['rot_w'].clip(-1,1))\ntest_temp['rot_angle']  = 2 * np.arccos(test_temp['rot_w'].clip(-1,1))\n\n# 4) Nhóm theo sequence_id và tổng hợp các tóm tắt gia tốc kế\nacc_agg_funcs = {\n    'acc_mag': ['mean', 'std', 'max']\n}\ntrain_acc_summary = train_temp.groupby('sequence_id').agg(acc_agg_funcs)\ntest_acc_summary  = test_temp.groupby('sequence_id').agg(acc_agg_funcs)\n\n# Làm phẳng cột MultiIndex\ntrain_acc_summary.columns = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\ntest_acc_summary.columns  = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\n\n# 5) Nhóm theo sequence_id và tổng hợp tóm tắt vòng quay\nrot_agg_funcs = {\n    'rot_angle': ['mean', 'std', 'max']\n}\ntrain_rot_summary = train_temp.groupby('sequence_id').agg(rot_agg_funcs)\ntest_rot_summary  = test_temp.groupby('sequence_id').agg(rot_agg_funcs)\n\ntrain_rot_summary.columns = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\ntest_rot_summary.columns  = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\n\n# 6) NHIỆT ĐỘ: năm cảm biến thm_1 … thm_5\nthm_cols = [f\"thm_{i}\" for i in range(1, 6)]\n\n# Xác định hàm tổng hợp: trung bình + độ lệch chuẩn\nthm_agg_funcs = {col: ['mean', 'std'] for col in thm_cols}\n\ntrain_thm_summary = train_temp.groupby('sequence_id').agg(thm_agg_funcs)\ntest_thm_summary  = test_temp.groupby('sequence_id').agg(thm_agg_funcs)\n\n# Làm phẳng các cột MultiIndex\nflattened_thm_cols = []\nfor sensor in thm_cols:\n    for stat in ['mean','std']:\n        flattened_thm_cols.append(f\"{sensor}_{stat}\")\n\ntrain_thm_summary.columns = flattened_thm_cols\ntest_thm_summary.columns  = flattened_thm_cols\n\n# 7) THỜI GIAN BAY: mỗi cảm biến i có 64 cột pixel: tof_i_v0 … tof_i_v63\n# Chúng ta sẽ tạo một “tof_i_mean_at_ts” cho mỗi dấu thời gian, sau đó tổng hợp theo từng chuỗi.\n\ndef compute_tof_sequence_summary(df):\n    # Khởi tạo một dict để giữ DataFrames theo từng chuỗi\n    seq_summaries = {}\n\n    for i in range(1, 6):\n        # Xây dựng danh sách các cột cho cảm biến i\n        tof_cols = [f\"tof_{i}_v{pix}\" for pix in range(64)]\n        # Thay thế -1 bằng NaN để chúng không làm lệch giá trị trung bình; chuyển đổi sang float\n        ts_grid = df[tof_cols].replace(-1, np.nan).astype(float)\n        # Tính toán “trung bình trên tất cả 64 pixel” cho mỗi dấu thời gian\n        df[f\"tof_{i}_mean_at_ts\"] = ts_grid.mean(axis=1)\n    \n   # Bây giờ, nhóm theo id chuỗi và tính giá trị trung bình và độ lệch chuẩn của các giá trị trung bình đó\n    agg_dict = {f\"tof_{i}_mean_at_ts\": ['mean','std'] for i in range(1, 6)}\n    summary = df.groupby('sequence_id').agg(agg_dict)\n    # Làm phẳng các cột MultiIndex\n    flat_cols = [f\"tof_{i}_{stat}\" for i in range(1, 6) for stat in ['mean','std']]\n    summary.columns = flat_cols\n    return summary\n\ntrain_tof_summary = compute_tof_sequence_summary(train_temp)\ntest_tof_summary  = compute_tof_sequence_summary(test_temp)\n\n# 8) Hợp nhất các tóm tắt accel, rotation, thm, tof (trên sequence_id)\ntrain_sensor_summary = (\n    train_acc_summary\n    .join(train_rot_summary, how='outer')\n    .join(train_thm_summary, how='outer')\n    .join(train_tof_summary, how='outer')\n)\n\ntest_sensor_summary = (\n    test_acc_summary\n    .join(test_rot_summary, how='outer')\n    .join(test_thm_summary, how='outer')\n    .join(test_tof_summary, how='outer')\n)\n\n# 9) Thêm cột “Dataset” để chúng ta có thể thực hiện box+hist song song\ntrain_sensor_summary['Dataset'] = 'Train'\ntest_sensor_summary['Dataset']  = 'Test'\n\n# 10) Ghép nối thành một DataFrame để vẽ đồ thị\ncombined_sensor_summary = pd.concat(\n    [train_sensor_summary, test_sensor_summary],\n    axis=0\n).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:08.521777Z","iopub.execute_input":"2025-08-18T14:26:08.522146Z","iopub.status.idle":"2025-08-18T14:26:19.749381Z","shell.execute_reply.started":"2025-08-18T14:26:08.522116Z","shell.execute_reply":"2025-08-18T14:26:19.747899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Target Feature Analysis (Univariate Analysis)","metadata":{}},{"cell_type":"markdown","source":"### Sensor‐Based Differentiation Across Demographics and Physiology Features","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# (1) Sáp nhập train_demographics vào train_df nếu chưa thực hiện\ntrain_df = train_df.merge(\n    train_dem_df,\n    on=\"subject\",\n    how=\"left\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:19.751198Z","iopub.execute_input":"2025-08-18T14:26:19.752046Z","iopub.status.idle":"2025-08-18T14:26:20.462522Z","shell.execute_reply.started":"2025-08-18T14:26:19.752018Z","shell.execute_reply":"2025-08-18T14:26:20.461499Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Xử lý missing values","metadata":{}},{"cell_type":"markdown","source":"## IMU – Gia tốc/quaternion (acc_*, rot_w/x/y/z)","metadata":{}},{"cell_type":"code","source":"def fix_imu(df):\n    imu_cols = [c for c in df.columns if c.startswith((\"acc_\", \"rot_\"))]\n    def _seq_fix(g):\n        g[imu_cols] = g[imu_cols].interpolate(limit_direction=\"both\")\n        # renormalize quaternion\n        q = g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]].to_numpy(float)\n        n = (q**2).sum(1)**0.5\n        n[n==0] = 1.0\n        q = q / n[:,None]\n        g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]] = q\n        return g\n    return df.groupby(\"sequence_id\", group_keys=False).apply(_seq_fix)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.466810Z","iopub.execute_input":"2025-08-18T14:26:20.467424Z","iopub.status.idle":"2025-08-18T14:26:20.474383Z","shell.execute_reply.started":"2025-08-18T14:26:20.467398Z","shell.execute_reply":"2025-08-18T14:26:20.473285Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Thermopile (thm_1..thm_5)","metadata":{}},{"cell_type":"code","source":"def fix_thm(df):\n    thm = [f\"thm_{i}\" for i in range(1,6)]\n    return df.groupby(\"sequence_id\", group_keys=False)[thm].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.475223Z","iopub.execute_input":"2025-08-18T14:26:20.476155Z","iopub.status.idle":"2025-08-18T14:26:20.500294Z","shell.execute_reply.started":"2025-08-18T14:26:20.476127Z","shell.execute_reply":"2025-08-18T14:26:20.499334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ToF – 5 cảm biến × 64 pixel (tof_i_vj)","metadata":{}},{"cell_type":"code","source":"def fix_tof_and_reduce(df):\n    # 1) sentinel -> NaN + flag\n    tof_pix = [c for c in df.columns if c.startswith(\"tof_\") and \"_v\" in c]\n    df[tof_pix] = df[tof_pix].replace(-1, np.nan)\n    # optional flags per sensor\n    for i in range(1,6):\n        cols = [f\"tof_{i}_v{p}\" for p in range(64) if f\"tof_{i}_v{p}\" in df]\n        df[f\"tof_{i}_missing_frac\"] = df[cols].isna().mean(axis=1)\n        df[f\"tof_{i}_mean\"] = df[cols].mean(axis=1)  # dimension reduction\n    # 2) interpolate the 5 means\n    mean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\n    df[mean_cols] = df.groupby(\"sequence_id\", group_keys=False)[mean_cols].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )\n    # 3) drop heavy pixel columns to save RAM\n    df.drop(columns=tof_pix, inplace=True)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.501402Z","iopub.execute_input":"2025-08-18T14:26:20.501716Z","iopub.status.idle":"2025-08-18T14:26:20.520563Z","shell.execute_reply.started":"2025-08-18T14:26:20.501691Z","shell.execute_reply":"2025-08-18T14:26:20.519477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pha/nhãn & metadata","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle_gesture = LabelEncoder().fit(train_df[\"gesture\"].astype(str))\ntrain_df[\"gesture_id\"] = le_gesture.transform(train_df[\"gesture\"].astype(str))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.521941Z","iopub.execute_input":"2025-08-18T14:26:20.522296Z","iopub.status.idle":"2025-08-18T14:26:20.662767Z","shell.execute_reply.started":"2025-08-18T14:26:20.522271Z","shell.execute_reply":"2025-08-18T14:26:20.661861Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pipeline tiền xử lý","metadata":{}},{"cell_type":"code","source":"def minimal_preprocess(df):\n    df = df.copy()\n    # 1) IMU\n    df = fix_imu(df)\n    # 2) Thermopile\n    thm_fixed = fix_thm(df)  # returns only thm cols\n    df[thm_fixed.columns] = thm_fixed\n    # 3) ToF -> means + flags, drop pixels\n    df = fix_tof_and_reduce(df)\n    # 4) Feature cơ bản (ví dụ):\n    df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n    df[\"rot_angle\"] = 2*np.arccos(np.clip(df[\"rot_w\"], -1, 1))\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.664063Z","iopub.execute_input":"2025-08-18T14:26:20.664387Z","iopub.status.idle":"2025-08-18T14:26:20.670482Z","shell.execute_reply.started":"2025-08-18T14:26:20.664359Z","shell.execute_reply":"2025-08-18T14:26:20.669431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration & Constants","metadata":{}},{"cell_type":"code","source":"target = \"gesture\"\ngrouper = \"subject\"\n\nmapper = {\n    \"Above ear - pull hair\": 0,\n    \"Cheek - pinch skin\": 1,\n    \"Eyebrow - pull hair\": 2,\n    \"Eyelash - pull hair\": 3, \n    \"Forehead - pull hairline\": 4,\n    \"Forehead - scratch\": 5,\n    \"Neck - pinch skin\": 6, \n    \"Neck - scratch\": 7,\n    \"Drink from bottle/cup\": 8,\n    \"Feel around in tray and pull out an object\": 9,\n    \"Glasses on/off\": 10,\n    \"Pinch knee/leg skin\": 11, \n    \"Pull air toward your face\": 12,\n    \"Scratch knee/leg skin\": 13,\n    \"Text on phone\": 14,\n    \"Wave hello\": 15,\n    \"Write name in air\": 16,\n    \"Write name on leg\": 17,\n}\n\ndef ScoreMetric(ytrue, ypreds) -> tuple:\n    \"\"\"CMI competition scoring metric\"\"\"\n    bscore = f1_score(\n        np.where(ytrue <= 7, 1, 0),\n        np.where(ypreds <= 7, 1, 0),\n        zero_division=0.0,\n    )\n    mscore = f1_score(\n        np.where(ytrue <= 7, ytrue, 99),\n        np.where(ypreds <= 7, ypreds, 99),\n        average=\"macro\", \n        zero_division=0.0,\n    )\n    return (0.5 * (bscore + mscore), bscore, mscore)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.671743Z","iopub.execute_input":"2025-08-18T14:26:20.672136Z","iopub.status.idle":"2025-08-18T14:26:20.698907Z","shell.execute_reply.started":"2025-08-18T14:26:20.672104Z","shell.execute_reply":"2025-08-18T14:26:20.697830Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Enhanced Feature Engineering","metadata":{}},{"cell_type":"code","source":"def make_ftre_unified_enhanced(sequence: pl.DataFrame, demographics: pl.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Enhanced feature extraction with BFRB-specific patterns\n    Total: ~220 dimensions (original 155 + 65 new features)\n    Designed to work with existing codebase architecture\n    \"\"\"\n    \n    agg_exprs = []\n    stat_cols = ['acc_x','acc_y','acc_z','rot_w','rot_x','rot_y','rot_z']\n    \n    # === EXISTING FEATURES (155 dimensions) - Keep unchanged ===\n    for c in stat_cols:\n        # Original features from the base implementation\n        agg_exprs += [\n            pl.col(c).mean().alias(f\"{c}_mean\"),\n            pl.col(c).std().alias(f\"{c}_std\"),\n            pl.col(c).var().alias(f\"{c}_var\"),\n            pl.col(c).quantile(0.25).alias(f\"{c}_q25\"),\n            pl.col(c).median().alias(f\"{c}_q50\"),\n            pl.col(c).quantile(0.75).alias(f\"{c}_q75\"),\n            pl.col(c).max().alias(f\"{c}_max\"),\n            pl.col(c).min().alias(f\"{c}_min\"),\n            pl.col(c).first().alias(f\"{c}_first\"),\n            pl.col(c).last().alias(f\"{c}_last\"),\n            (pl.col(c).last() - pl.col(c).first()).alias(f\"{c}_delta\"),\n            pl.corr(\"sequence_counter\", c).alias(f\"{c}_corr_time\"),\n            pl.col(c).diff().mean().alias(f\"{c}_diff_mean\"),\n            pl.col(c).diff().std().alias(f\"{c}_diff_std\"),\n            pl.col(c).skew().alias(f\"{c}_skew\"),\n            pl.col(c).kurtosis().alias(f\"{c}_kurt\"),\n            pl.col(c).diff().abs().gt(0).sum().alias(f\"{c}_n_changes\")\n        ]\n        \n        # Segment features from original\n        agg_exprs += [\n            pl.when(pl.col(\"sequence_counter\") < 0.1 * pl.max(\"sequence_counter\"))\n              .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg1_mean\"),\n            pl.when(pl.col(\"sequence_counter\") > 0.9 * pl.max(\"sequence_counter\"))\n              .then(pl.col(c)).otherwise(None).mean().alias(f\"{c}_seg3_mean\"),\n        ]\n        \n        # Additional quantiles from original\n        agg_exprs += [\n            pl.col(c).quantile(0.25, \"nearest\").alias(f\"{c}_t25\"),\n            pl.col(c).quantile(0.75, \"nearest\").alias(f\"{c}_t75\"),\n        ]\n    \n    # === NEW ENHANCED FEATURES ===\n    \n    # 1. BFRB-specific movement patterns (7 features per stat_col = 49 total)\n    for c in stat_cols:\n        agg_exprs += [\n            # Peak detection - count significant changes\n            pl.col(c).diff().abs().gt(pl.col(c).std() * 0.5).sum().alias(f\"{c}_peak_count\"),\n            \n            # Movement intensity - mean absolute deviation\n            (pl.col(c) - pl.col(c).mean()).abs().mean().alias(f\"{c}_mad\"),\n            \n            # Range normalization - signal range relative to std\n            ((pl.col(c).max() - pl.col(c).min()) / (pl.col(c).std() + 1e-8)).alias(f\"{c}_range_norm\"),\n            \n            # Acceleration analysis - second derivative\n            pl.col(c).diff().diff().abs().mean().alias(f\"{c}_accel_mean\"),\n            pl.col(c).diff().diff().std().alias(f\"{c}_accel_std\"),\n            \n            # Movement consistency - coefficient of variation\n            (pl.col(c).std() / (pl.col(c).mean().abs() + 1e-8)).alias(f\"{c}_cv\"),\n            \n            # Energy measure - sum of squared values\n            (pl.col(c) ** 2).sum().alias(f\"{c}_energy\"),\n        ]\n    \n    # 2. Multi-axis interaction features (15 features)\n    # Acceleration magnitude and patterns\n    agg_exprs += [\n        # 3D acceleration magnitude\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().mean().alias(\"acc_mag_mean\"),\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().std().alias(\"acc_mag_std\"),\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().max().alias(\"acc_mag_max\"),\n        (pl.col(\"acc_x\")**2 + pl.col(\"acc_y\")**2 + pl.col(\"acc_z\")**2).sqrt().min().alias(\"acc_mag_min\"),\n        \n        # 3D rotation magnitude\n        (pl.col(\"rot_x\")**2 + pl.col(\"rot_y\")**2 + pl.col(\"rot_z\")**2).sqrt().mean().alias(\"rot_mag_mean\"),\n        (pl.col(\"rot_x\")**2 + pl.col(\"rot_y\")**2 + pl.col(\"rot_z\")**2).sqrt().std().alias(\"rot_mag_std\"),\n        \n        # Axis dominance - which axis has most movement\n        (pl.col(\"acc_x\").abs().mean() / (pl.col(\"acc_x\").abs().mean() + pl.col(\"acc_y\").abs().mean() + pl.col(\"acc_z\").abs().mean() + 1e-8)).alias(\"acc_x_dominance\"),\n        (pl.col(\"acc_y\").abs().mean() / (pl.col(\"acc_x\").abs().mean() + pl.col(\"acc_y\").abs().mean() + pl.col(\"acc_z\").abs().mean() + 1e-8)).alias(\"acc_y_dominance\"),\n        (pl.col(\"acc_z\").abs().mean() / (pl.col(\"acc_x\").abs().mean() + pl.col(\"acc_y\").abs().mean() + pl.col(\"acc_z\").abs().mean() + 1e-8)).alias(\"acc_z_dominance\"),\n        \n        # Cross-axis correlations (key pairs only)\n        pl.corr(\"acc_x\", \"acc_y\").alias(\"acc_xy_corr\"),\n        pl.corr(\"acc_y\", \"acc_z\").alias(\"acc_yz_corr\"),\n        pl.corr(\"acc_x\", \"acc_z\").alias(\"acc_xz_corr\"),\n        \n        # Rotation-acceleration relationship\n        pl.corr(\"acc_x\", \"rot_x\").alias(\"acc_rot_x_corr\"),\n        pl.corr(\"acc_y\", \"rot_y\").alias(\"acc_rot_y_corr\"),\n        pl.corr(\"acc_z\", \"rot_z\").alias(\"acc_rot_z_corr\"),\n    ]\n    \n    # 3. Temporal pattern analysis (14 features)\n    agg_exprs += [\n        # Sequence characteristics\n        pl.max(\"sequence_counter\").alias(\"seq_length\"),\n        pl.col(\"sequence_counter\").count().alias(\"seq_count\"),\n        \n        # Early vs late behavior comparison for acceleration\n        (pl.when(pl.col(\"sequence_counter\") < 0.3 * pl.max(\"sequence_counter\"))\n         .then(pl.col(\"acc_x\")).otherwise(None).std() / \n         (pl.when(pl.col(\"sequence_counter\") > 0.7 * pl.max(\"sequence_counter\"))\n          .then(pl.col(\"acc_x\")).otherwise(None).std() + 1e-8)).alias(\"acc_x_early_late_ratio\"),\n        \n        (pl.when(pl.col(\"sequence_counter\") < 0.3 * pl.max(\"sequence_counter\"))\n         .then(pl.col(\"acc_y\")).otherwise(None).std() / \n         (pl.when(pl.col(\"sequence_counter\") > 0.7 * pl.max(\"sequence_counter\"))\n          .then(pl.col(\"acc_y\")).otherwise(None).std() + 1e-8)).alias(\"acc_y_early_late_ratio\"),\n        \n        (pl.when(pl.col(\"sequence_counter\") < 0.3 * pl.max(\"sequence_counter\"))\n         .then(pl.col(\"acc_z\")).otherwise(None).std() / \n         (pl.when(pl.col(\"sequence_counter\") > 0.7 * pl.max(\"sequence_counter\"))\n          .then(pl.col(\"acc_z\")).otherwise(None).std() + 1e-8)).alias(\"acc_z_early_late_ratio\"),\n        \n        # Movement intensity over time\n        (pl.col(\"acc_x\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"acc_x_intensity_rate\"),\n        (pl.col(\"acc_y\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"acc_y_intensity_rate\"),\n        (pl.col(\"acc_z\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"acc_z_intensity_rate\"),\n        \n        # Rotation intensity over time  \n        (pl.col(\"rot_x\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"rot_x_intensity_rate\"),\n        (pl.col(\"rot_y\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"rot_y_intensity_rate\"),\n        (pl.col(\"rot_z\").diff().abs().sum() / (pl.max(\"sequence_counter\") + 1)).alias(\"rot_z_intensity_rate\"),\n        \n        # Total movement complexity\n        ((pl.col(\"acc_x\").diff().abs().sum() + pl.col(\"acc_y\").diff().abs().sum() + pl.col(\"acc_z\").diff().abs().sum()) / \n         (pl.max(\"sequence_counter\") + 1)).alias(\"total_acc_complexity\"),\n        \n        ((pl.col(\"rot_x\").diff().abs().sum() + pl.col(\"rot_y\").diff().abs().sum() + pl.col(\"rot_z\").diff().abs().sum()) / \n         (pl.max(\"sequence_counter\") + 1)).alias(\"total_rot_complexity\"),\n        \n        # Movement smoothness (inverse of jerk)\n        pl.col(\"acc_x\").diff().diff().abs().sum().alias(\"acc_x_jerk\"),\n        pl.col(\"acc_y\").diff().diff().abs().sum().alias(\"acc_y_jerk\"),\n    ]\n    \n    # === EXISTING PROCESSING (unchanged) ===\n    result = (\n        sequence.\n        group_by(pl.col([\"sequence_id\", \"subject\"]), maintain_order=True).\n        agg(agg_exprs).\n        select(pl.all().shrink_dtype()).\n        join(\n            demographics.select(pl.all().shrink_dtype()),\n            how=\"left\",\n            on=[\"subject\"],\n        ).\n        drop([\"subject\"], strict=False).\n        to_pandas()\n    )\n    \n    # Handle NaN values with more robust approach\n    result = result.fillna(0.0)\n    \n    # Remove any potential infinite values\n    result = result.replace([np.inf, -np.inf], 0.0)\n    \n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.700022Z","iopub.execute_input":"2025-08-18T14:26:20.700359Z","iopub.status.idle":"2025-08-18T14:26:20.737189Z","shell.execute_reply.started":"2025-08-18T14:26:20.700328Z","shell.execute_reply":"2025-08-18T14:26:20.736163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Loading & Preprocessing","metadata":{}},{"cell_type":"code","source":"print(\"Loading data...\")\ntrain = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\ntraind = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\ntest = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\ntestd = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")\n\n# Select IMU-only columns (consistent with both notebooks)\nsel_cols = [c for c in test.collect_schema().names() if \"thm\" not in c and \"tof\" not in c]\n\nprint(\"Extracting enhanced features (~220 dimensions)...\")\nXtrain = make_ftre_unified_enhanced(train.select(pl.col(sel_cols)), traind)\nXtest = make_ftre_unified_enhanced(test.select(pl.col(sel_cols)), testd)\n\nprint(f\"Enhanced training features shape: {Xtrain.shape}\")\nprint(f\"Enhanced test features shape: {Xtest.shape}\")\n\n# Prepare targets\nYtrain = train.select(pl.col([\"sequence_id\", \"subject\", target])).unique().to_pandas()\nYtrain[target] = Ytrain[target].map(mapper).astype(np.int32)\nYtrain = Xtrain[[\"sequence_id\"]].merge(Ytrain, how=\"inner\", on=[\"sequence_id\"])\nytrain = Ytrain[target].values\n\nprint(f\"Target distribution:\")\nprint(pd.Series(ytrain).value_counts().sort_index())\n\n# Cross-validation setup (consistent with both notebooks)\ncv = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n\nygrp = np.zeros(len(Xtrain))\nfor fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, Ytrain[target], Ytrain[grouper])):\n    ygrp[dev_idx] = fold_nb\n\ncv = PredefinedSplit(ygrp)\nprint(f\"Cross-validation setup complete. Shapes: {Xtrain.shape}, {ytrain.shape}, {Xtest.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:20.738323Z","iopub.execute_input":"2025-08-18T14:26:20.738650Z","iopub.status.idle":"2025-08-18T14:26:35.360947Z","shell.execute_reply.started":"2025-08-18T14:26:20.738623Z","shell.execute_reply":"2025-08-18T14:26:35.360034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TabM Classifier","metadata":{}},{"cell_type":"code","source":"class TabMClassifier:\n    \"\"\"TabM Classifier for CMI BFRB task - Enhanced with unified features\"\"\"\n    \n    def __init__(\n        self,\n        arch_type: str = 'tabm-mini',\n        backbone: dict = {'type': 'MLP', 'n_blocks': 3, 'd_block': 256, 'dropout': 0.15},\n        d_embedding: int = 64,\n        bin_count: int = 52,\n        k: int = 8,\n        learning_rate: float = 1e-3,\n        weight_decay: float = 1e-4,\n        max_epochs: int = 100,\n        patience: int = 20,\n        batch_size: int = 32,\n        device: Optional[str] = None,\n        random_state: int = 42,\n        verbose: bool = True\n    ):\n        self.arch_type = arch_type\n        self.backbone = backbone\n        self.d_embedding = d_embedding\n        self.bin_count = bin_count\n        self.k = k\n        self.learning_rate = learning_rate\n        self.weight_decay = weight_decay\n        self.max_epochs = max_epochs\n        self.patience = patience\n        self.batch_size = batch_size\n        self.device = torch.device(device if device else ('cuda' if torch.cuda.is_available() else 'cpu'))\n        self.random_state = random_state\n        self.verbose = verbose\n        self.n_classes = 18\n        \n    def fit(self, X: pd.DataFrame, y: np.array, eval_set: Tuple[pd.DataFrame, np.array]):\n        \"\"\"Fit TabM classifier with enhanced features\"\"\"\n        \n        # Preprocess data\n        X_cat_train, X_cont_train, cat_cardinalities, y_train = self._preprocess_data(X, y, training=True)\n        X_cat_val, X_cont_val, _, y_val = self._preprocess_data(eval_set[0], eval_set[1], training=False)\n        \n        if self.verbose:\n            print(f\"Training data shapes: X_cont: {X_cont_train.shape}, X_cat: {X_cat_train.shape}\")\n            print(f\"Validation data shapes: X_cont: {X_cont_val.shape}, X_cat: {X_cat_val.shape}\")\n        \n        # Create model\n        bins = None\n        if self.arch_type == 'tabm-mini' and X_cont_train.shape[1] > 0:\n            try:\n                bins = rtdl_num_embeddings.compute_bins(X_cont_train, n_bins=self.bin_count)\n            except Exception as e:\n                if self.verbose:\n                    print(f\"Warning: Could not compute bins, using None: {e}\")\n                bins = None\n        \n        self.model = Model(\n            n_num_features=X_cont_train.shape[1],\n            cat_cardinalities=cat_cardinalities,\n            n_classes=self.n_classes,\n            backbone=self.backbone,\n            bins=bins,\n            num_embeddings=(\n                None if bins is None else {\n                    'type': 'PiecewiseLinearEmbeddings',\n                    'd_embedding': self.d_embedding,\n                    'activation': False,\n                    'version': 'B',\n                }\n            ),\n            arch_type=self.arch_type,\n            k=self.k,\n        ).to(self.device)\n        \n        optimizer = torch.optim.AdamW(\n            make_parameter_groups(self.model), \n            lr=self.learning_rate, \n            weight_decay=self.weight_decay\n        )\n        \n        loss_fn = torch.nn.CrossEntropyLoss().to(self.device)\n        \n        # Training loop\n        best = {\n            'epoch': -1,\n            'eval_score': -np.inf,\n            'model_state_dict': None,\n        }\n        remaining_patience = self.patience\n        \n        for epoch in range(self.max_epochs):\n            # Training phase\n            self.model.train()\n            train_losses = []\n            \n            indices = torch.randperm(len(y_train), device=self.device).split(self.batch_size)\n            \n            for batch_idx in indices:\n                optimizer.zero_grad()\n                \n                try:\n                    if X_cat_train.shape[1] > 0:\n                        logits = self.model(X_cont_train[batch_idx], X_cat_train[batch_idx])\n                    else:\n                        logits = self.model(X_cont_train[batch_idx], None)\n                    \n                    logits_flat = logits.contiguous().reshape(-1, self.n_classes)\n                    y_repeated = y_train[batch_idx].repeat_interleave(self.k)\n                    \n                    loss = loss_fn(logits_flat, y_repeated)\n                    \n                except Exception as e:\n                    print(f\"Error in forward pass: {e}\")\n                    raise e\n                \n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)\n                optimizer.step()\n                \n                train_losses.append(loss.item())\n            \n            # Validation phase\n            self.model.eval()\n            val_preds = []\n            val_true = []\n            \n            with torch.no_grad():\n                for batch_idx in range(0, len(y_val), self.batch_size):\n                    end_idx = min(batch_idx + self.batch_size, len(y_val))\n                    \n                    if X_cat_val.shape[1] > 0:\n                        logits = self.model(\n                            X_cont_val[batch_idx:end_idx],\n                            X_cat_val[batch_idx:end_idx]\n                        )\n                    else:\n                        logits = self.model(\n                            X_cont_val[batch_idx:end_idx],\n                            None\n                        )\n                    \n                    logits = logits.contiguous()\n                    probs = torch.softmax(logits, dim=-1).mean(dim=1)\n                    preds = probs.argmax(dim=-1).cpu().numpy()\n                    \n                    val_preds.extend(preds)\n                    val_true.extend(y_val[batch_idx:end_idx].cpu().numpy())\n            \n            # Calculate CMI score\n            score, bscore, mscore = ScoreMetric(np.array(val_true), np.array(val_preds))\n            \n            if self.verbose:\n                print(f'Epoch {epoch} | Train Loss: {np.mean(train_losses):.4f} | '\n                      f'Val Score: {score:.4f} | Binary F1: {bscore:.4f} | Macro F1: {mscore:.4f}')\n            \n            # Check for improvement\n            if score > best['eval_score']:\n                best['epoch'] = epoch\n                best['eval_score'] = score\n                best['model_state_dict'] = self.model.state_dict()\n                remaining_patience = self.patience\n                \n                if self.verbose:\n                    print('🌸 New best epoch! 🌸')\n            else:\n                remaining_patience -= 1\n            \n            # Early stopping\n            if remaining_patience == 0:\n                if self.verbose:\n                    print(f\"Early stopping at epoch {epoch}\")\n                break\n        \n        # Restore best model\n        self.model.load_state_dict(best['model_state_dict'])\n        if self.verbose:\n            print(f\"Best model restored from epoch {best['epoch']} with score {best['eval_score']:.4f}\")\n        \n    def predict(self, X: pd.DataFrame, batch_size: Optional[int] = 1024) -> np.ndarray:\n        \"\"\"Predict classes\"\"\"\n        X_cat, X_cont, _, _ = self._preprocess_data(X, y=None, training=False)\n        \n        self.model.eval()\n        predictions = []\n        \n        with torch.no_grad():\n            for batch_idx in range(0, len(X), batch_size):\n                end_idx = min(batch_idx + batch_size, len(X))\n                \n                if X_cat.shape[1] > 0:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        X_cat[batch_idx:end_idx]\n                    )\n                else:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        None\n                    )\n                \n                logits = logits.contiguous()\n                probs = torch.softmax(logits, dim=-1).mean(dim=1)\n                preds = probs.argmax(dim=-1).cpu().numpy()\n                \n                predictions.extend(preds)\n        \n        return np.array(predictions)\n    \n    def predict_proba(self, X: pd.DataFrame, batch_size: Optional[int] = 1024) -> np.ndarray:\n        \"\"\"Predict class probabilities\"\"\"\n        X_cat, X_cont, _, _ = self._preprocess_data(X, y=None, training=False)\n        \n        self.model.eval()\n        probabilities = []\n        \n        with torch.no_grad():\n            for batch_idx in range(0, len(X), batch_size):\n                end_idx = min(batch_idx + batch_size, len(X))\n                \n                if X_cat.shape[1] > 0:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        X_cat[batch_idx:end_idx]\n                    )\n                else:\n                    logits = self.model(\n                        X_cont[batch_idx:end_idx],\n                        None\n                    )\n                \n                logits = logits.contiguous()\n                probs = torch.softmax(logits, dim=-1).mean(dim=1).cpu().numpy()\n                probabilities.append(probs)\n        \n        return np.vstack(probabilities)\n    \n    def _preprocess_data(self, X: pd.DataFrame, y: Optional[np.array], training: bool):\n        \"\"\"Preprocess data for TabM\"\"\"\n        \n        # Handle non-constant columns\n        if training:\n            self._non_constant_columns = X.columns[X.nunique() > 1]\n        \n        X = X[self._non_constant_columns].copy()\n        \n        # Separate categorical and continuous features\n        categorical_features = ['adult_child', 'sex', 'handedness'] if 'adult_child' in X.columns else []\n        \n        if categorical_features:\n            X_cat = X[categorical_features].to_numpy()\n        else:\n            X_cat = np.empty((len(X), 0))\n            \n        X_cont = X.drop(columns=categorical_features, errors='ignore').to_numpy().astype(np.float32)\n        \n        # Handle categorical features\n        cat_cardinalities = []\n        if X_cat.shape[1] > 0:\n            if training:\n                self._categorical_encoders = []\n                for i, col in enumerate(categorical_features):\n                    encoder = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)\n                    self._categorical_encoders.append(encoder)\n                    cat_cardinalities.append(X[col].nunique() + 1)\n            \n            X_cat_encoded = []\n            for i, encoder in enumerate(self._categorical_encoders):\n                if training:\n                    encoded = encoder.fit_transform(X_cat[:, i:i+1])\n                else:\n                    encoded = encoder.transform(X_cat[:, i:i+1])\n                X_cat_encoded.append(encoded)\n            \n            if X_cat_encoded:\n                X_cat = np.concatenate(X_cat_encoded, axis=1).astype(np.int64)\n            else:\n                X_cat = np.empty((len(X), 0), dtype=np.int64)\n        else:\n            X_cat = np.empty((len(X), 0), dtype=np.int64)\n        \n        # Handle continuous features\n        if training:\n            if X_cont.shape[1] > 0:\n                noise = np.random.default_rng(0).normal(0.0, 1e-5, X_cont.shape).astype(X_cont.dtype)\n                self._cont_feature_preprocessor = MinMaxScaler().fit(X_cont + noise)\n            else:\n                self._cont_feature_preprocessor = MinMaxScaler()\n        \n        if X_cont.shape[1] > 0:\n            X_cont = self._cont_feature_preprocessor.transform(X_cont)\n        \n        # Convert to tensors\n        X_cat = torch.tensor(X_cat, dtype=torch.long, device=self.device)\n        X_cont = torch.tensor(X_cont, dtype=torch.float32, device=self.device)\n        \n        if y is not None:\n            y = torch.tensor(y, dtype=torch.long, device=self.device)\n        \n        return X_cat, X_cont, cat_cardinalities, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:35.362202Z","iopub.execute_input":"2025-08-18T14:26:35.362534Z","iopub.status.idle":"2025-08-18T14:26:35.399619Z","shell.execute_reply.started":"2025-08-18T14:26:35.362503Z","shell.execute_reply":"2025-08-18T14:26:35.398652Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TabM Training Pipeline","metadata":{}},{"cell_type":"code","source":"def train_tabm_models():\n    \"\"\"Train 5 TabM models with enhanced features\"\"\"\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"TRAINING TABM MODELS (5-FOLD CV) - ENHANCED FEATURES\")\n    print(\"=\"*70)\n    \n    tabm_oof_preds = []\n    tabm_fitted_models = []\n    drop_cols = [\"sequence_id\", target, grouper]\n    \n    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain), start=1):\n        print(f\"\\n{'='*50}\")\n        print(f\"Training TabM Fold {fold_nb}/5\")\n        print(f\"{'='*50}\")\n        \n        Xtr = Xtrain.iloc[train_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        Xdev = Xtrain.iloc[dev_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        ytr = ytrain[train_idx]\n        ydev = ytrain[dev_idx]\n        \n        print(f\"TabM Fold {fold_nb} - Train: {len(Xtr)}, Val: {len(Xdev)}\")\n        \n        # Initialize and train model\n        model = TabMClassifier(\n            arch_type='tabm-mini',\n            backbone={'type': 'MLP', 'n_blocks': 3, 'd_block': 256, 'dropout': 0.15},\n            d_embedding=64,\n            k=8,\n            learning_rate=1e-3,\n            weight_decay=1e-4,\n            max_epochs=30,\n            patience=8,\n            batch_size=16,\n            verbose=True\n        )\n        \n        try:\n            model.fit(Xtr, ytr, eval_set=(Xdev, ydev))\n            tabm_fitted_models.append(model)\n            \n            # Get predictions\n            dev_preds = model.predict(Xdev)\n            tabm_oof_preds.append(pd.DataFrame({'pred': dev_preds}, index=dev_idx))\n            \n            # Calculate score\n            score, bscore, mscore = ScoreMetric(ydev, dev_preds)\n            print(f\"\\nTabM Fold {fold_nb} Results:\")\n            print(f\"Overall Score: {score:.6f}\")\n            print(f\"Binary F1: {bscore:.6f}\")\n            print(f\"Macro F1: {mscore:.6f}\")\n            \n        except Exception as e:\n            print(f\"Error in TabM fold {fold_nb}: {e}\")\n            print(\"Skipping this fold...\")\n            continue\n    \n    if not tabm_fitted_models:\n        raise RuntimeError(\"No TabM models were successfully trained!\")\n    \n    # Combine OOF predictions\n    tabm_oof_preds = pd.concat(tabm_oof_preds, axis=0).sort_index()\n    \n    # Calculate overall score\n    overall_score, overall_bscore, overall_mscore = ScoreMetric(ytrain, tabm_oof_preds['pred'].values)\n    print(f\"\\n{'='*50}\")\n    print(f\"TABM OVERALL RESULTS - ENHANCED\")\n    print(f\"{'='*50}\")\n    print(f\"Overall Score: {overall_score:.6f}\")\n    print(f\"Binary F1: {overall_bscore:.6f}\")\n    print(f\"Macro F1: {overall_mscore:.6f}\")\n    \n    return tabm_fitted_models, tabm_oof_preds\n\n# Execute TabM training\ntabm_models, tabm_oof = train_tabm_models()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:26:35.400931Z","iopub.execute_input":"2025-08-18T14:26:35.401266Z","iopub.status.idle":"2025-08-18T15:17:50.962052Z","shell.execute_reply.started":"2025-08-18T14:26:35.401237Z","shell.execute_reply":"2025-08-18T15:17:50.960547Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CatBoost Training Pipeline","metadata":{}},{"cell_type":"code","source":"def train_catboost_models():\n    \"\"\"Train 5 CatBoost models with enhanced features\"\"\"\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"TRAINING CATBOOST MODELS (5-FOLD CV) - ENHANCED FEATURES\")\n    print(\"=\"*70)\n    \n    catboost_model = CBC(\n        loss_function=\"MultiClass\",\n        task_type=\"CPU\" if torch.cuda.is_available() == False else \"GPU\",\n        iterations=10000,\n        max_depth=8,\n        learning_rate=0.03,\n        colsample_bylevel=0.50 if torch.cuda.is_available() == False else None,\n        random_state=42,\n        early_stopping_rounds=200,\n        l2_leaf_reg=0.40,\n        verbose=0,\n    )\n    \n    catboost_oof_preds = []\n    catboost_fitted_models = []\n    drop_cols = [\"sequence_id\", target, grouper]\n    \n    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain), start=1):\n        print(f\"\\nTraining CatBoost Fold {fold_nb}/5\")\n        \n        Xtr = Xtrain.iloc[train_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        Xdev = Xtrain.iloc[dev_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        ytr = ytrain[train_idx]\n        ydev = ytrain[dev_idx]\n        \n        model = clone(catboost_model)\n        model.fit(\n            Xtr, ytr, eval_set=[(Xdev, ydev)], verbose=0\n        )\n        dev_preds = pd.DataFrame(model.predict_proba(Xdev), index=dev_idx)\n        catboost_oof_preds.append(dev_preds)\n        catboost_fitted_models.append(model)\n        \n        score, bscore, mscore = ScoreMetric(\n            ydev.flatten(), \n            dev_preds.idxmax(axis=1).values.flatten()\n        )\n        print(f\"CatBoost Fold {fold_nb} - Score: {score:.6f} | Binary F1: {bscore:.6f} | Macro F1: {mscore:.6f}\")\n    \n    catboost_oof_preds = pd.concat(catboost_oof_preds, axis=0).sort_index(ascending=True)\n    \n    score, bscore, mscore = ScoreMetric(\n        ytrain.flatten(), \n        catboost_oof_preds.idxmax(axis=1).values.flatten()\n    )\n    print(f\"\\n{'='*50}\")\n    print(f\"CATBOOST OVERALL RESULTS - ENHANCED\")\n    print(f\"{'='*50}\")\n    print(f\"Overall Score: {score:.6f}\")\n    print(f\"Binary F1: {bscore:.6f}\")\n    print(f\"Macro F1: {mscore:.6f}\")\n    \n    collect()\n    return catboost_fitted_models, catboost_oof_preds\n\n# Execute CatBoost training\ncatboost_models, catboost_oof = train_catboost_models()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T15:17:50.964139Z","iopub.execute_input":"2025-08-18T15:17:50.966009Z","iopub.status.idle":"2025-08-18T17:42:28.174555Z","shell.execute_reply.started":"2025-08-18T15:17:50.965972Z","shell.execute_reply":"2025-08-18T17:42:28.173368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble & Validation","metadata":{}},{"cell_type":"code","source":"def validate_ensemble_strategies():\n    \"\"\"Test different ensemble strategies with proper probability handling\"\"\"\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"ENSEMBLE STRATEGY VALIDATION - ENHANCED FEATURES\")\n    print(\"=\"*70)\n    \n    # First, let's get TabM probabilities properly\n    print(\"Getting TabM probabilities from trained models...\")\n    tabm_probs_oof = []\n    drop_cols = [\"sequence_id\", target, grouper]\n    \n    for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain)):\n        Xdev = Xtrain.iloc[dev_idx].drop(drop_cols, axis=1, errors=\"ignore\")\n        try:\n            # Get probabilities from the trained TabM model\n            dev_probs = tabm_models[fold_nb].predict_proba(Xdev)\n            tabm_probs_oof.append(pd.DataFrame(dev_probs, index=dev_idx))\n        except Exception as e:\n            print(f\"Warning: Could not get probabilities from TabM model {fold_nb}, using predictions: {e}\")\n            # Fallback: use soft probabilities instead of hard one-hot\n            dev_preds = tabm_models[fold_nb].predict(Xdev) \n            soft_probs = np.zeros((len(dev_preds), 18))\n            soft_probs[np.arange(len(dev_preds)), dev_preds] = 0.8  # 80% confidence\n            # Distribute remaining 20% equally among other classes\n            for i in range(len(dev_preds)):\n                remaining = 0.2 / 17  # Distribute among 17 other classes\n                soft_probs[i, :] += remaining\n                soft_probs[i, dev_preds[i]] = 0.8  # Reset main prediction to 80%\n            tabm_probs_oof.append(pd.DataFrame(soft_probs, index=dev_idx))\n    \n    tabm_probs_oof = pd.concat(tabm_probs_oof, axis=0).sort_index()\n    \n    print(\"TabM probabilities shape:\", tabm_probs_oof.shape)\n    print(\"CatBoost probabilities shape:\", catboost_oof.shape)\n    \n    # Strategy 1: Equal weight (0.5:0.5) - NOW WITH REAL PROBABILITIES\n    def equal_weight_ensemble(tabm_probs, catboost_probs):\n        ensemble_probs = 0.5 * tabm_probs + 0.5 * catboost_probs\n        return np.argmax(ensemble_probs, axis=1)\n    \n    # Strategy 2: Performance-weighted ensemble\n    def weighted_ensemble(tabm_probs, catboost_probs, tabm_weight=0.7):\n        ensemble_probs = tabm_weight * tabm_probs + (1-tabm_weight) * catboost_probs\n        return np.argmax(ensemble_probs, axis=1)\n    \n    # Strategy 3: TabM only (from probabilities)\n    def tabm_only(tabm_probs, catboost_probs):\n        return np.argmax(tabm_probs, axis=1)\n    \n    # Strategy 4: CatBoost only\n    def catboost_only(tabm_probs, catboost_probs):\n        return np.argmax(catboost_probs, axis=1)\n    \n    strategies = {\n        \"TabM Only\": tabm_only,\n        \"CatBoost Only\": catboost_only,\n        \"Equal Weight (0.5:0.5)\": equal_weight_ensemble,\n        \"Weighted (0.3:0.7)\": lambda t, c: weighted_ensemble(t, c, 0.3),\n        \"Weighted (0.2:0.8)\": lambda t, c: weighted_ensemble(t, c, 0.2),\n        \"Weighted (0.4:0.6)\": lambda t, c: weighted_ensemble(t, c, 0.4),\n    }\n    \n    results = {}\n    \n    for strategy_name, strategy_func in strategies.items():\n        ensemble_preds = strategy_func(tabm_probs_oof.values, catboost_oof.values)\n        score, bscore, mscore = ScoreMetric(ytrain, ensemble_preds)\n        results[strategy_name] = {\n            'score': score,\n            'binary_f1': bscore,\n            'macro_f1': mscore\n        }\n        print(f\"{strategy_name:25} - Score: {score:.6f} | Binary F1: {bscore:.6f} | Macro F1: {mscore:.6f}\")\n    \n    # Find best strategy\n    best_strategy = max(results.keys(), key=lambda k: results[k]['score'])\n    best_score = results[best_strategy]['score']\n    \n    print(f\"\\n🏆 Best Strategy: {best_strategy} with score {best_score:.6f}\")\n    \n    return best_strategy, strategies[best_strategy], tabm_probs_oof\n\n# Validate ensemble strategies with proper probabilities\nbest_strategy_name, best_strategy_func, tabm_probs_oof = validate_ensemble_strategies()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T17:42:28.176151Z","iopub.execute_input":"2025-08-18T17:42:28.176444Z","iopub.status.idle":"2025-08-18T17:42:36.289902Z","shell.execute_reply.started":"2025-08-18T17:42:28.176420Z","shell.execute_reply":"2025-08-18T17:42:36.288938Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Final Ensemble Prediction Function","metadata":{}},{"cell_type":"code","source":"def create_ensemble_prediction_function(tabm_models, catboost_models, strategy_func):\n    \"\"\"Create unified prediction function with real probability ensemble\"\"\"\n    \n    def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n        \"\"\"Ensemble prediction function with real probabilities\"\"\"\n        try:\n            # Extract enhanced features\n            drop_cols = [\"sequence_id\", target, grouper]\n            Xtest = make_ftre_unified_enhanced(sequence, demographics).drop(drop_cols, axis=1, errors=\"ignore\")\n            \n            # Get TabM probabilities (average across 5 models)\n            tabm_probs_list = []\n            for model in tabm_models:\n                try:\n                    pred_probs = model.predict_proba(Xtest)\n                    tabm_probs_list.append(pred_probs)\n                except Exception as e:\n                    print(f\"Warning: TabM predict_proba failed: {e}, using soft prediction\")\n                    # Soft prediction fallback\n                    pred = model.predict(Xtest)\n                    soft_probs = np.zeros((len(Xtest), 18))\n                    soft_probs[np.arange(len(Xtest)), pred] = 0.8\n                    remaining = 0.2 / 17\n                    soft_probs += remaining\n                    soft_probs[np.arange(len(Xtest)), pred] = 0.8\n                    tabm_probs_list.append(soft_probs)\n            \n            # Average TabM probabilities\n            avg_tabm_probs = np.mean(tabm_probs_list, axis=0)  # Shape: (1, 18)\n            \n            # Get CatBoost probabilities (average across 5 models)\n            catboost_probs_list = []\n            for model in catboost_models:\n                pred_probs = model.predict_proba(Xtest)\n                catboost_probs_list.append(pred_probs)\n            \n            # Average CatBoost probabilities\n            avg_catboost_probs = np.mean(catboost_probs_list, axis=0)  # Shape: (1, 18)\n            \n            # Apply ensemble strategy with REAL probabilities\n            final_pred = strategy_func(avg_tabm_probs, avg_catboost_probs)[0]\n            \n            # Individual predictions for logging\n            tabm_pred = np.argmax(avg_tabm_probs, axis=1)[0]\n            catboost_pred = np.argmax(avg_catboost_probs, axis=1)[0]\n            \n            # Convert back to gesture name\n            gesture_name = {v: k for k, v in mapper.items()}[final_pred]\n            \n            print(f\"TabM pred: {tabm_pred} (conf: {avg_tabm_probs[0, tabm_pred]:.3f})\")\n            print(f\"CatBoost pred: {catboost_pred} (conf: {avg_catboost_probs[0, catboost_pred]:.3f})\")\n            print(f\"Ensemble pred: {final_pred} -> {gesture_name}\")\n            \n            return gesture_name\n            \n        except Exception as e:\n            print(f\"Error in ensemble prediction: {e}\")\n            import traceback\n            traceback.print_exc()\n            return \"Forehead - scratch\"\n    \n    return predict\n\n# Create final prediction function with guaranteed ensemble\nensemble_predict_fn = create_ensemble_prediction_function(tabm_models, catboost_models, best_strategy_func)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T17:42:36.290894Z","iopub.execute_input":"2025-08-18T17:42:36.291244Z","iopub.status.idle":"2025-08-18T17:42:36.302739Z","shell.execute_reply.started":"2025-08-18T17:42:36.291222Z","shell.execute_reply":"2025-08-18T17:42:36.301723Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Kaggle Submission Setup","metadata":{}},{"cell_type":"code","source":"import kaggle_evaluation.cmi_inference_server\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"SETTING UP KAGGLE SUBMISSION - ENHANCED FEATURES\")\nprint(\"=\"*70)\nprint(f\"Total TabM models: {len(tabm_models)}\")\nprint(f\"Total CatBoost models: {len(catboost_models)}\")\nprint(f\"Total ensemble models: {len(tabm_models) + len(catboost_models)}\")\nprint(f\"Best ensemble strategy: {best_strategy_name}\")\nprint(f\"Feature dimensions: ~220 (enhanced)\")\n\n# Setup inference server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(ensemble_predict_fn)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    print(\"Running in competition mode...\")\n    inference_server.serve()\nelse:\n    print(\"Running in local test mode...\")\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T17:42:36.303756Z","iopub.execute_input":"2025-08-18T17:42:36.304105Z","iopub.status.idle":"2025-08-18T17:42:37.960855Z","shell.execute_reply.started":"2025-08-18T17:42:36.304078Z","shell.execute_reply":"2025-08-18T17:42:37.960118Z"}},"outputs":[],"execution_count":null}]}