{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":242561727,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings\nfrom matplotlib.ticker import MaxNLocator\nfrom mpl_toolkits.mplot3d import Axes3D\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nfrom sklearn.metrics import accuracy_score, f1_score\nimport joblib\nfrom scipy.spatial.transform import Rotation as R","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:48:29.496392Z","iopub.execute_input":"2025-08-18T07:48:29.496670Z","iopub.status.idle":"2025-08-18T07:48:31.669711Z","shell.execute_reply.started":"2025-08-18T07:48:29.496644Z","shell.execute_reply":"2025-08-18T07:48:31.669106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tải tập dữ liệu\n#Dữ liệu cảm biến cho tập huấn luyện (Train Sensor Data): \ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\n#Thông tin nhân khẩu học của đối tượng trong tập huấn luyện (giới tính, tuổi, ...): \ntrain_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\n\n#Dữ liệu cảm biến cho tập kiểm tra:\ntest_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\n\n# Thông tin nhân khẩu học của đối tượng trong tập kiểm tra:\ntest_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:48:31.670538Z","iopub.execute_input":"2025-08-18T07:48:31.670904Z","iopub.status.idle":"2025-08-18T07:49:05.021521Z","shell.execute_reply.started":"2025-08-18T07:48:31.670884Z","shell.execute_reply":"2025-08-18T07:49:05.020737Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.1 Khám phá dữ liệu","metadata":{}},{"cell_type":"markdown","source":"### 2.1.3 Tóm tắt thống kê các tính năng ","metadata":{}},{"cell_type":"code","source":"datasets = {\n    \"Train Data\": train_df,\n    \"Train Demographics\": train_dem_df,\n    \"Test Data\": test_df,\n    \"Test Demographics\": test_dem_df,\n}\n\nfor name, df in datasets.items():\n    print(f\"Missing values in {name}:\")\n    missing = df.isnull().sum()\n    missing = missing[missing > 0]\n    if not missing.empty:\n        print(missing)\n    else:\n        print(\"  ✅ No missing values.\")\n    print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:05.023769Z","iopub.execute_input":"2025-08-18T07:49:05.024060Z","iopub.status.idle":"2025-08-18T07:49:05.906869Z","shell.execute_reply.started":"2025-08-18T07:49:05.024039Z","shell.execute_reply":"2025-08-18T07:49:05.905972Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.5 kiểm tra giá trị trùng lặp","metadata":{}},{"cell_type":"code","source":"#Kiểm tra trùng lặp (Duplicate Rows): .duplicated(): Trả về Series boolean, dòng nào trùng lặp sẽ là True.\n#                                     .sum(): Đếm tổng số dòng trùng lặp.\n#Kiểm tra xem dữ liệu có bản ghi nào bị trùng không — quan trọng cho việc làm sạch dữ liệu.\n\n# Đếm các hàng trùng lặp trong train_df\ntrain_duplicates = train_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong  test_df\ntest_duplicates = test_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong train_dem_df (optional)\ntrain_dem_duplicates = train_dem_df.duplicated().sum()\n# Đếm các hàng trùng lặp trong test_dem_df (optional)\ntest_dem_duplicates = test_dem_df.duplicated().sum()\n\n# In số lượng dòng trùng:\nprint(f\"Number of duplicate rows in train_df: {train_duplicates}\")\nprint(f\"Number of duplicate rows in test_df: {test_duplicates}\")\nprint(f\"Number of duplicate rows in train_dem_df: {train_dem_duplicates}\")\nprint(f\"Number of duplicate rows in test_dem_df: {test_dem_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:05.907857Z","iopub.execute_input":"2025-08-18T07:49:05.908370Z","iopub.status.idle":"2025-08-18T07:49:10.806471Z","shell.execute_reply.started":"2025-08-18T07:49:05.908337Z","shell.execute_reply":"2025-08-18T07:49:10.805561Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"markdown","source":"### Tỷ lệ phần trăm Null","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport gc\nimport pickle\nimport os\nfrom pathlib import Path\nfrom types import SimpleNamespace\nimport shutil\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.amp import autocast, GradScaler\nfrom torch.optim import lr_scheduler, Adam, AdamW\n\nfrom glob import glob\nfrom sklearn.model_selection import KFold, StratifiedGroupKFold\nfrom metric import score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:10.807379Z","iopub.execute_input":"2025-08-18T07:49:10.807766Z","iopub.status.idle":"2025-08-18T07:49:14.985305Z","shell.execute_reply.started":"2025-08-18T07:49:10.807744Z","shell.execute_reply":"2025-08-18T07:49:14.984540Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### UTILS/HELPERS","metadata":{}},{"cell_type":"code","source":"def null_percent(df):\n    per=((df.isnull().sum()/len(df))*100).round(2)\n    return per[per>0]\n\nprint(\"Nan Values in Train data\")\nprint(null_percent(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:14.986031Z","iopub.execute_input":"2025-08-18T07:49:14.986397Z","iopub.status.idle":"2025-08-18T07:49:15.851299Z","shell.execute_reply.started":"2025-08-18T07:49:14.986378Z","shell.execute_reply":"2025-08-18T07:49:15.850589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Quan sát hoặc xu hướng ban đầu","metadata":{}},{"cell_type":"code","source":"# --- 2.4 Hợp nhất dữ liệu nhân khẩu học và cảm biến ---\nmerged_train = pd.merge(train_df, train_dem_df, on=\"subject\", how=\"left\")\nmerged_test = pd.merge(test_df, test_dem_df, on=\"subject\", how=\"left\")\n\nprint(\"Merged Train Shape:\", merged_train.shape)\nprint(\"Merged Test Shape:\", merged_test.shape)\ndisplay(merged_train.head(2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:15.852001Z","iopub.execute_input":"2025-08-18T07:49:15.852207Z","iopub.status.idle":"2025-08-18T07:49:16.536568Z","shell.execute_reply.started":"2025-08-18T07:49:15.852189Z","shell.execute_reply":"2025-08-18T07:49:16.535953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Thống kê mô tả toàn bộ merged_train ---\nmerged_train.describe().T\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:16.537465Z","iopub.execute_input":"2025-08-18T07:49:16.537743Z","iopub.status.idle":"2025-08-18T07:49:25.482340Z","shell.execute_reply.started":"2025-08-18T07:49:16.537718Z","shell.execute_reply":"2025-08-18T07:49:25.481436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra thiếu giá trị & thống kê cảm biến: \n#Danh sách các cột không thuộc cảm biến (có thể là ID, thông tin khác). \n\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nsensor_cols = [col for col in train_df.columns if not col.startswith(excluded_prefixes)]\n\n# Sensor Data Summary for TRAIN\n#isnull().sum(): Đếm số giá trị bị thiếu.\nmissing_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    '[TRAIN] Missing Count': train_df[sensor_cols].isnull().sum().values,\n    '[TRAIN] Missing %': (train_df[sensor_cols].isnull().sum().values / len(train_df)) * 100\n})\n\n#nunique(): Đếm số lượng giá trị duy nhất.\nunique_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Unique Values [TRAIN]': train_df[sensor_cols].nunique().values\n})\n\n#dtypes: Lấy kiểu dữ liệu của từng cột.\ndtypes_sensor = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Data Type': train_df[sensor_cols].dtypes.values\n})\n\n# Merge all summaries (NO test set)\n#merge: Gộp các bảng thống kê thành bảng duy nhất theo Feature.\nsensor_summary = missing_sensor_train \\\n    .merge(unique_sensor_train, on='Feature', how='left') \\\n    .merge(dtypes_sensor, on='Feature', how='left')\n\n# Display styled DataFrame (mask NaNs just for styling)\n#fillna(0): Điền giá trị thiếu bằng 0 (cho đẹp mắt khi hiển thị).\n#.style.background_gradient: Tô màu nền theo giá trị giúp dễ nhìn.\nstyled_df = sensor_summary.fillna(0)\nstyled_df.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:25.486326Z","iopub.execute_input":"2025-08-18T07:49:25.486595Z","iopub.status.idle":"2025-08-18T07:49:26.593368Z","shell.execute_reply.started":"2025-08-18T07:49:25.486575Z","shell.execute_reply":"2025-08-18T07:49:26.592489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Thống kê tương tự cho nhân khẩu học: Tương tự như phần thống kê cảm biến nhưng áp dụng cho dữ liệu nhân khẩu học.\n\n# Cột nhân khẩu học (không loại trừ)\ndem_cols = train_dem_df.columns\n\n# Giá trị bị thiếu trong dữ liệu nhân khẩu học của train \nmissing_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    '[TRAIN DEMO] Missing Count': train_dem_df[dem_cols].isnull().sum().values,\n    '[TRAIN DEMO] Missing %': (train_dem_df[dem_cols].isnull().sum().values / len(train_dem_df)) * 100\n})\n\n# Giá trị duy nhất được tính trong dữ liệu nhân khẩu học của train \nunique_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    'Unique Values [TRAIN DEMO]': train_dem_df[dem_cols].nunique().values\n})\n\n# Data types\ndtypes_demo = pd.DataFrame({\n    'Feature': dem_cols,\n    'Data Type': train_dem_df[dem_cols].dtypes.values\n})\n\n# Tóm tắt hợp nhất (chỉ dành cho đào tạo)\ndemo_summary = (\n    missing_demo_train\n    .merge(unique_demo_train, on='Feature', how='left')\n    .merge(dtypes_demo, on='Feature', how='left')\n)\n\n# Hiển thị tóm tắt theo phong cách\ndemo_summary.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:26.594489Z","iopub.execute_input":"2025-08-18T07:49:26.595030Z","iopub.status.idle":"2025-08-18T07:49:26.622022Z","shell.execute_reply.started":"2025-08-18T07:49:26.595000Z","shell.execute_reply":"2025-08-18T07:49:26.621130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.2.2. Sensor-Based Features (Tính năng dựa trên cảm biến)","metadata":{}},{"cell_type":"markdown","source":"# 3. EDA","metadata":{}},{"cell_type":"markdown","source":"## 3.1. Numerical Feature Distribution Analysis (Phân tích phân phối tính năng số) (Phân tích đơn biến)","metadata":{}},{"cell_type":"markdown","source":"### 3.1.1 Participant & Contextual Features (Tính năng của người tham gia và bối cảnh)","metadata":{}},{"cell_type":"markdown","source":"### 3.1.2 Sensor-Based Features (Tính năng dựa trên cảm biến)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# 1) Sao chép đào tạo và kiểm tra để chúng ta không sửa đổi DataFrame gốc\ntrain_temp = train_df.copy()\ntest_temp  = test_df.copy()\n\n# 2) GIA TỐC KẾ: tính toán độ lớn tại mỗi dấu thời gian\ntrain_temp['acc_mag'] = np.sqrt(\n    train_temp['acc_x']**2 + train_temp['acc_y']**2 + train_temp['acc_z']**2\n)\ntest_temp['acc_mag'] = np.sqrt(\n    test_temp['acc_x']**2 + test_temp['acc_y']**2 + test_temp['acc_z']**2\n)\n\n# 3) ROTATION: tính toán “góc quay” từ thành phần quaternion w\n# (Lưu ý: rot_w nằm trong [-1,1], do đó arccos hợp lệ. Chúng tôi bỏ qua NaN nếu có.)\ntrain_temp['rot_angle'] = 2 * np.arccos(train_temp['rot_w'].clip(-1,1))\ntest_temp['rot_angle']  = 2 * np.arccos(test_temp['rot_w'].clip(-1,1))\n\n# 4) Nhóm theo sequence_id và tổng hợp các tóm tắt gia tốc kế\nacc_agg_funcs = {\n    'acc_mag': ['mean', 'std', 'max']\n}\ntrain_acc_summary = train_temp.groupby('sequence_id').agg(acc_agg_funcs)\ntest_acc_summary  = test_temp.groupby('sequence_id').agg(acc_agg_funcs)\n\n# Làm phẳng cột MultiIndex\ntrain_acc_summary.columns = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\ntest_acc_summary.columns  = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\n\n# 5) Nhóm theo sequence_id và tổng hợp tóm tắt vòng quay\nrot_agg_funcs = {\n    'rot_angle': ['mean', 'std', 'max']\n}\ntrain_rot_summary = train_temp.groupby('sequence_id').agg(rot_agg_funcs)\ntest_rot_summary  = test_temp.groupby('sequence_id').agg(rot_agg_funcs)\n\ntrain_rot_summary.columns = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\ntest_rot_summary.columns  = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\n\n# 6) NHIỆT ĐỘ: năm cảm biến thm_1 … thm_5\nthm_cols = [f\"thm_{i}\" for i in range(1, 6)]\n\n# Xác định hàm tổng hợp: trung bình + độ lệch chuẩn\nthm_agg_funcs = {col: ['mean', 'std'] for col in thm_cols}\n\ntrain_thm_summary = train_temp.groupby('sequence_id').agg(thm_agg_funcs)\ntest_thm_summary  = test_temp.groupby('sequence_id').agg(thm_agg_funcs)\n\n# Làm phẳng các cột MultiIndex\nflattened_thm_cols = []\nfor sensor in thm_cols:\n    for stat in ['mean','std']:\n        flattened_thm_cols.append(f\"{sensor}_{stat}\")\n\ntrain_thm_summary.columns = flattened_thm_cols\ntest_thm_summary.columns  = flattened_thm_cols\n\n# 7) THỜI GIAN BAY: mỗi cảm biến i có 64 cột pixel: tof_i_v0 … tof_i_v63\n# Chúng ta sẽ tạo một “tof_i_mean_at_ts” cho mỗi dấu thời gian, sau đó tổng hợp theo từng chuỗi.\n\ndef compute_tof_sequence_summary(df):\n    # Khởi tạo một dict để giữ DataFrames theo từng chuỗi\n    seq_summaries = {}\n\n    for i in range(1, 6):\n        # Xây dựng danh sách các cột cho cảm biến i\n        tof_cols = [f\"tof_{i}_v{pix}\" for pix in range(64)]\n        # Thay thế -1 bằng NaN để chúng không làm lệch giá trị trung bình; chuyển đổi sang float\n        ts_grid = df[tof_cols].replace(-1, np.nan).astype(float)\n        # Tính toán “trung bình trên tất cả 64 pixel” cho mỗi dấu thời gian\n        df[f\"tof_{i}_mean_at_ts\"] = ts_grid.mean(axis=1)\n    \n   # Bây giờ, nhóm theo id chuỗi và tính giá trị trung bình và độ lệch chuẩn của các giá trị trung bình đó\n    agg_dict = {f\"tof_{i}_mean_at_ts\": ['mean','std'] for i in range(1, 6)}\n    summary = df.groupby('sequence_id').agg(agg_dict)\n    # Làm phẳng các cột MultiIndex\n    flat_cols = [f\"tof_{i}_{stat}\" for i in range(1, 6) for stat in ['mean','std']]\n    summary.columns = flat_cols\n    return summary\n\ntrain_tof_summary = compute_tof_sequence_summary(train_temp)\ntest_tof_summary  = compute_tof_sequence_summary(test_temp)\n\n# 8) Hợp nhất các tóm tắt accel, rotation, thm, tof (trên sequence_id)\ntrain_sensor_summary = (\n    train_acc_summary\n    .join(train_rot_summary, how='outer')\n    .join(train_thm_summary, how='outer')\n    .join(train_tof_summary, how='outer')\n)\n\ntest_sensor_summary = (\n    test_acc_summary\n    .join(test_rot_summary, how='outer')\n    .join(test_thm_summary, how='outer')\n    .join(test_tof_summary, how='outer')\n)\n\n# 9) Thêm cột “Dataset” để chúng ta có thể thực hiện box+hist song song\ntrain_sensor_summary['Dataset'] = 'Train'\ntest_sensor_summary['Dataset']  = 'Test'\n\n# 10) Ghép nối thành một DataFrame để vẽ đồ thị\ncombined_sensor_summary = pd.concat(\n    [train_sensor_summary, test_sensor_summary],\n    axis=0\n).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:26.622963Z","iopub.execute_input":"2025-08-18T07:49:26.623278Z","iopub.status.idle":"2025-08-18T07:49:32.413202Z","shell.execute_reply.started":"2025-08-18T07:49:26.623251Z","shell.execute_reply":"2025-08-18T07:49:32.412528Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3 Target Feature Analysis (Univariate Analysis)","metadata":{}},{"cell_type":"markdown","source":"### 3.4.2 Sensor‐Based Differentiation Across Demographics and Physiology Features","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# (1) Sáp nhập train_demographics vào train_df nếu chưa thực hiện\ntrain_df = train_df.merge(\n    train_dem_df,\n    on=\"subject\",\n    how=\"left\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:32.414070Z","iopub.execute_input":"2025-08-18T07:49:32.414344Z","iopub.status.idle":"2025-08-18T07:49:33.060303Z","shell.execute_reply.started":"2025-08-18T07:49:32.414322Z","shell.execute_reply":"2025-08-18T07:49:33.059651Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Xử lý missing values","metadata":{}},{"cell_type":"markdown","source":"### 1. IMU – Gia tốc/quaternion (acc_*, rot_w/x/y/z)","metadata":{}},{"cell_type":"code","source":"def fix_imu(df):\n    imu_cols = [c for c in df.columns if c.startswith((\"acc_\", \"rot_\"))]\n    def _seq_fix(g):\n        g[imu_cols] = g[imu_cols].interpolate(limit_direction=\"both\")\n        # renormalize quaternion\n        q = g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]].to_numpy(float)\n        n = (q**2).sum(1)**0.5\n        n[n==0] = 1.0\n        q = q / n[:,None]\n        g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]] = q\n        return g\n    return df.groupby(\"sequence_id\", group_keys=False).apply(_seq_fix)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.061037Z","iopub.execute_input":"2025-08-18T07:49:33.061254Z","iopub.status.idle":"2025-08-18T07:49:33.067418Z","shell.execute_reply.started":"2025-08-18T07:49:33.061238Z","shell.execute_reply":"2025-08-18T07:49:33.066478Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. Thermopile (thm_1..thm_5)","metadata":{}},{"cell_type":"code","source":"def fix_thm(df):\n    thm = [f\"thm_{i}\" for i in range(1,6)]\n    return df.groupby(\"sequence_id\", group_keys=False)[thm].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.068332Z","iopub.execute_input":"2025-08-18T07:49:33.068588Z","iopub.status.idle":"2025-08-18T07:49:33.088930Z","shell.execute_reply.started":"2025-08-18T07:49:33.068565Z","shell.execute_reply":"2025-08-18T07:49:33.088190Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. ToF – 5 cảm biến × 64 pixel (tof_i_vj)","metadata":{}},{"cell_type":"code","source":"def fix_tof_and_reduce(df):\n    # 1) sentinel -> NaN + flag\n    tof_pix = [c for c in df.columns if c.startswith(\"tof_\") and \"_v\" in c]\n    df[tof_pix] = df[tof_pix].replace(-1, np.nan)\n    # optional flags per sensor\n    for i in range(1,6):\n        cols = [f\"tof_{i}_v{p}\" for p in range(64) if f\"tof_{i}_v{p}\" in df]\n        df[f\"tof_{i}_missing_frac\"] = df[cols].isna().mean(axis=1)\n        df[f\"tof_{i}_mean\"] = df[cols].mean(axis=1)  # dimension reduction\n    # 2) interpolate the 5 means\n    mean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\n    df[mean_cols] = df.groupby(\"sequence_id\", group_keys=False)[mean_cols].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )\n    # 3) drop heavy pixel columns to save RAM\n    df.drop(columns=tof_pix, inplace=True)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.090000Z","iopub.execute_input":"2025-08-18T07:49:33.090265Z","iopub.status.idle":"2025-08-18T07:49:33.104473Z","shell.execute_reply.started":"2025-08-18T07:49:33.090241Z","shell.execute_reply":"2025-08-18T07:49:33.103676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 4. Pha/nhãn & metadata","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle_gesture = LabelEncoder().fit(train_df[\"gesture\"].astype(str))\ntrain_df[\"gesture_id\"] = le_gesture.transform(train_df[\"gesture\"].astype(str))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.105256Z","iopub.execute_input":"2025-08-18T07:49:33.105540Z","iopub.status.idle":"2025-08-18T07:49:33.223994Z","shell.execute_reply.started":"2025-08-18T07:49:33.105516Z","shell.execute_reply":"2025-08-18T07:49:33.223184Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5. Pipeline tiền xử lý","metadata":{}},{"cell_type":"code","source":"def minimal_preprocess(df):\n    df = df.copy()\n    # 1) IMU\n    df = fix_imu(df)\n    # 2) Thermopile\n    thm_fixed = fix_thm(df)  # returns only thm cols\n    df[thm_fixed.columns] = thm_fixed\n    # 3) ToF -> means + flags, drop pixels\n    df = fix_tof_and_reduce(df)\n    # 4) Feature cơ bản (ví dụ):\n    df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n    df[\"rot_angle\"] = 2*np.arccos(np.clip(df[\"rot_w\"], -1, 1))\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.224773Z","iopub.execute_input":"2025-08-18T07:49:33.225019Z","iopub.status.idle":"2025-08-18T07:49:33.230463Z","shell.execute_reply.started":"2025-08-18T07:49:33.225004Z","shell.execute_reply":"2025-08-18T07:49:33.229705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\ncfg = SimpleNamespace(**{})\n\ncfg.fname = 'CMI-001'\ncfg.seed = 42\ncfg.device = torch.device('cuda')\n\ncfg.input_path = Path('/kaggle/input/cmi-detect-behavior-with-sensor-data')\ncfg.save_path = Path('./saved_models')\n\nimc_features = ['acc_x', 'acc_y', 'acc_z']\nall_features = [\n    'acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z', \n    'thm_1', 'thm_2', 'thm_3','thm_4', 'thm_5'\n]\ncfg.imc_index = [0,1,2,3,4,5,6]\ncfg.all_index = [0,1,2,3,4,5,6,7,8,9,10,11] \n# choose which two feature to generate extra features\ncfg.gen_index = [\n    [0, 1],\n    [0, 2],\n    [1, 2]\n]\n# these sequences have only imc features\nimc_only_seq_ids = ['SEQ_000389', 'SEQ_000993', 'SEQ_001905', 'SEQ_002559', 'SEQ_003071', 'SEQ_003221', 'SEQ_004536', 'SEQ_006554', 'SEQ_006816', 'SEQ_007924', 'SEQ_008584', 'SEQ_011032', 'SEQ_012168', 'SEQ_012170', 'SEQ_012619', 'SEQ_013381', 'SEQ_013964', 'SEQ_015093', 'SEQ_016489', 'SEQ_016690', 'SEQ_017041', 'SEQ_017397', 'SEQ_017550', 'SEQ_017726', 'SEQ_018069', 'SEQ_019829', 'SEQ_020153', 'SEQ_020185', 'SEQ_020377', 'SEQ_021335', 'SEQ_022409', 'SEQ_022726', 'SEQ_022927', 'SEQ_024987', 'SEQ_025240', 'SEQ_025685', 'SEQ_027154', 'SEQ_028775', 'SEQ_028871', 'SEQ_029693', 'SEQ_032705', 'SEQ_033061', 'SEQ_033922', 'SEQ_034359', 'SEQ_034994', 'SEQ_035090', 'SEQ_035116', 'SEQ_035309', 'SEQ_035419', 'SEQ_035699', 'SEQ_038052', 'SEQ_039253', 'SEQ_040910', 'SEQ_041113', 'SEQ_041199', 'SEQ_041541', 'SEQ_042079', 'SEQ_042145', 'SEQ_042187', 'SEQ_043744', 'SEQ_044004', 'SEQ_044143', 'SEQ_044475', 'SEQ_045927', 'SEQ_045995', 'SEQ_046244', 'SEQ_047721', 'SEQ_048233', 'SEQ_048511', 'SEQ_048861', 'SEQ_049663', 'SEQ_050020', 'SEQ_050738', 'SEQ_052603', 'SEQ_052680', 'SEQ_052722', 'SEQ_053884', 'SEQ_054261', 'SEQ_055297', 'SEQ_055346', 'SEQ_056071', 'SEQ_056418', 'SEQ_056430', 'SEQ_057193', 'SEQ_057886', 'SEQ_058311', 'SEQ_058580', 'SEQ_058729', 'SEQ_058753', 'SEQ_058923', 'SEQ_059569', 'SEQ_060041', 'SEQ_060392', 'SEQ_061589', 'SEQ_064578', 'SEQ_065179']\n\ncfg.max_length = 188\ncfg.lr = 1e-3\ncfg.num_epochs = 75\ncfg.batch_size = 128\ncfg.workers = 4\ncfg.grad_value = 2.0\ncfg.accumulate = 1\ncfg.decay = 0.01\ncfg.no_decay = True\n\ntargets = [\n    'Above ear - pull hair',\n    'Cheek - pinch skin',\n    'Eyebrow - pull hair',\n    'Eyelash - pull hair',\n    'Forehead - pull hairline',\n    'Forehead - scratch',\n    'Neck - pinch skin',\n    'Neck - scratch',\n    'Drink from bottle/cup',\n    'Feel around in tray and pull out an object',\n    'Glasses on/off',\n    'Pinch knee/leg skin',\n    'Pull air toward your face',\n    'Scratch knee/leg skin',\n    'Text on phone',\n    'Wave hello',\n    'Write name in air',\n    'Write name on leg'\n]\ncfg.label2id = {}\nfor i, k in enumerate(targets):\n    cfg.label2id[k] = i\n\ncfg.num_classes = len(targets)\ncfg.targets = np.array(targets)\n\nrandom.seed(cfg.seed)\nnp.random.seed(cfg.seed)\ntorch.manual_seed(cfg.seed)\ntorch.cuda.manual_seed(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.231191Z","iopub.execute_input":"2025-08-18T07:49:33.231482Z","iopub.status.idle":"2025-08-18T07:49:33.259839Z","shell.execute_reply.started":"2025-08-18T07:49:33.231463Z","shell.execute_reply":"2025-08-18T07:49:33.259261Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## load data","metadata":{}},{"cell_type":"code","source":"demo = pd.read_csv(cfg.input_path / \"train_demographics.csv\")\ntrain = pd.read_csv(cfg.input_path / \"train.csv\")\n\ntrain_agg = train.groupby(['sequence_type','sequence_id','subject','gesture'])['acc_x'].agg(['count']).reset_index()\ntrain_agg = train_agg.rename(columns={'count': 'seq_length'})\ndemo = pd.merge(train_agg, demo, on='subject', how='left').sort_values(['subject','sequence_id']).reset_index(drop=True)\nprint(f\"We have {len(demo)} sequences for training.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:33.260541Z","iopub.execute_input":"2025-08-18T07:49:33.260778Z","iopub.status.idle":"2025-08-18T07:49:55.634650Z","shell.execute_reply.started":"2025-08-18T07:49:33.260754Z","shell.execute_reply":"2025-08-18T07:49:55.633943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kfold = StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=42)\ndemo['fold'] = -1\nlayers = demo.sequence_type.astype(str) + '/' + demo.handedness.astype(str) + '/' + demo.sex.astype(str)\nfor i, (tr_idx, va_idx) in enumerate(kfold.split(demo, y=layers, groups=demo.subject)):\n    demo.loc[va_idx, 'fold'] = i\n\n# f_name is the file path for Dataset\ndemo['f_name'] = demo['subject'].astype(str) + '_' + demo['sequence_id'].astype(str)\ndemo.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:55.635567Z","iopub.execute_input":"2025-08-18T07:49:55.635906Z","iopub.status.idle":"2025-08-18T07:49:55.712652Z","shell.execute_reply.started":"2025-08-18T07:49:55.635878Z","shell.execute_reply":"2025-08-18T07:49:55.711704Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## calculate the mean/std for normalization","metadata":{}},{"cell_type":"code","source":"for fold in range(4):\n    train_ids = set(demo.loc[demo.fold!=fold, \"sequence_id\"].unique())\n    temp = train[train.sequence_id.isin(train_ids)]\n    print(f\"Fold {fold}, Size: {len(train_ids)}\")\n    all_features_mean = np.nanmean(temp[all_features], axis=0)\n    all_features_std = np.nanstd(temp[all_features], axis=0)\n    np.save(f\"all_features_mean_fold{fold}.npy\", all_features_mean)\n    np.save(f\"all_features_std_fold{fold}.npy\", all_features_std)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:55.713910Z","iopub.execute_input":"2025-08-18T07:49:55.714146Z","iopub.status.idle":"2025-08-18T07:49:58.219338Z","shell.execute_reply.started":"2025-08-18T07:49:55.714127Z","shell.execute_reply":"2025-08-18T07:49:58.218482Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## save input array for each sequence","metadata":{}},{"cell_type":"code","source":"file_path = Path(\"input_arrays\")\nif file_path.exists():\n    shutil.rmtree(file_path)\nfile_path.mkdir(exist_ok=True)\n\nfor i, df in tqdm(train.groupby('sequence_id')):\n    subject = df['subject'].values[0]\n    sequence_id = df['sequence_id'].values[0]\n    for col in all_features:\n        df[col] = df[col].ffill().bfill().fillna(0.)\n    f_name = f\"{subject}_{sequence_id}.npy\"\n    np.save(file_path / f_name, df[all_features].values.astype(np.float32))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:49:58.220172Z","iopub.execute_input":"2025-08-18T07:49:58.220492Z","iopub.status.idle":"2025-08-18T07:50:39.060009Z","shell.execute_reply.started":"2025-08-18T07:49:58.220447Z","shell.execute_reply":"2025-08-18T07:50:39.059125Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## some functions","metadata":{}},{"cell_type":"code","source":"def pad_sequence(arr, max_len):\n    \"\"\"\n    pad sequences to a fixed length\n    \"\"\"\n    target_shape = (max_len, arr.shape[1])\n    if arr.shape[0] > max_len:\n        return arr[:max_len, :]\n\n    pad_length = max_len - arr.shape[0]\n    padded_arr = np.zeros(target_shape, dtype=arr.dtype)\n    padded_arr[:arr.shape[0], :] = arr\n    return padded_arr\n\n\ndef feature_gen(arr, gen_index):\n    \"\"\"\n    use two features to generate more features\n    for example, acc_x - acc_y --> acc_x_y_diff\n    \"\"\"\n    target_shape = (arr.shape[0], arr.shape[1]+len(gen_index))\n    gen_arr = np.zeros(target_shape, dtype=arr.dtype)\n    gen_arr[:, :arr.shape[1]] = arr\n    for i, ind in enumerate(gen_index):\n        gen_arr[:, arr.shape[1]+i] = arr[:, ind[0]] - arr[:, ind[1]]\n    return gen_arr\n\n\ndef get_optimizer(model, cfg):\n    no_decay = [\"bias\", \"norm\"]\n    if cfg.no_decay:\n        optimizer_parameters = [\n            {'params': [p for n, p in model.named_parameters() \n                        if not any(nd in n for nd in no_decay)],\n             'weight_decay': cfg.decay},\n            {'params': [p for n, p in model.named_parameters() \n                        if any(nd in n for nd in no_decay)],\n             'weight_decay': 0.0},\n        ]\n    else:\n        optimizer_parameters = model.parameters()\n    optimizer = torch.optim.AdamW(optimizer_parameters, lr=cfg.lr)\n    return optimizer\n\n\ndef get_dataloader(dataset, cfg, istrain):\n    dataloader = DataLoader(\n        dataset, \n        batch_size=cfg.batch_size,\n        num_workers=cfg.workers,\n        shuffle=istrain,\n        pin_memory=False,\n        drop_last=istrain\n    )\n    return dataloader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:50:39.060996Z","iopub.execute_input":"2025-08-18T07:50:39.061306Z","iopub.status.idle":"2025-08-18T07:50:39.070000Z","shell.execute_reply.started":"2025-08-18T07:50:39.061278Z","shell.execute_reply":"2025-08-18T07:50:39.069181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CMIDataset(Dataset):\n    def __init__(self, df, cfg, feat_mean, feat_std, feature_index):\n        self.cfg = cfg\n        self.targets = df[\"sequence_type\"].values\n        self.labels = df[\"gesture\"].values\n        self.f_names = df[\"f_name\"].values\n        self.seq_lengths = df[\"seq_length\"].values\n        self.label2id = cfg.label2id\n        self.feat_mean = feat_mean\n        self.feat_std = feat_std\n        self.feature_index = feature_index\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        inputs = np.load(f'/kaggle/working/input_arrays/{self.f_names[idx]}.npy')\n        inputs = (inputs - self.feat_mean) / self.feat_std\n        inputs = inputs[:, self.feature_index]\n        seq_len = self.seq_lengths[idx]\n        \n        if len(inputs) > self.cfg.max_length:\n            inputs = inputs[:self.cfg.max_length, :]\n            seq_len = self.cfg.max_length\n        if len(inputs) < self.cfg.max_length:\n            inputs = pad_sequence(inputs, self.cfg.max_length)\n            seq_len = self.cfg.max_length\n\n        inputs = feature_gen(inputs, self.cfg.gen_index)\n        return {\n            'features': torch.from_numpy(inputs).float(),\n            'targets': torch.tensor(self.label2id[self.labels[idx]], dtype=torch.long),\n            'seq_len': torch.tensor(seq_len, dtype=torch.long)\n        }\n\n\nclass CNN1D(nn.Module):\n    def __init__(self, cfg, feature_dim):\n        super().__init__()\n        self.max_length = cfg.max_length\n        self.feature_dim = feature_dim\n        self.num_classes = cfg.num_classes\n\n        self.block1 = nn.Sequential(\n            nn.Conv1d(self.feature_dim, 256, kernel_size=7, padding=3),\n            nn.BatchNorm1d(256),\n            nn.GELU(),\n            nn.MaxPool1d(kernel_size=2),\n            nn.Dropout(0.3)\n        )\n        self.block2 = nn.Sequential(\n            nn.Conv1d(256, 384, kernel_size=5, padding=2),\n            nn.BatchNorm1d(384),\n            nn.GELU(),\n            nn.MaxPool1d(kernel_size=2),\n            nn.Dropout(0.3)\n        )\n        self.block3 = nn.Sequential(\n            nn.Conv1d(384, 512, kernel_size=3, padding=1),\n            nn.BatchNorm1d(512),\n            nn.GELU(),\n            nn.MaxPool1d(kernel_size=2),\n            nn.Dropout(0.4)\n        )\n        self._to_linear = None\n        self._calculate_conv_output((self.feature_dim, self.max_length))\n\n        self.classifier = nn.Sequential(\n            nn.Linear(self._to_linear, 512),\n            nn.GELU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, 256),\n            nn.GELU(),\n            nn.Dropout(0.4),\n            nn.Linear(256, self.num_classes)\n        )\n    \n    def _calculate_conv_output(self, shape):\n        with torch.no_grad():\n            sample = torch.zeros(1, *shape)\n            sample = self.block1(sample)\n            sample = self.block2(sample)\n            sample = self.block3(sample)\n            self._to_linear = sample.view(1, -1).size(1)\n    \n    def forward(self, x):\n        # remember to switch seq_length and num_features in Pytorch!\n        x = x.permute(0, 2, 1)\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:50:39.071003Z","iopub.execute_input":"2025-08-18T07:50:39.071266Z","iopub.status.idle":"2025-08-18T07:50:39.091577Z","shell.execute_reply.started":"2025-08-18T07:50:39.071248Z","shell.execute_reply":"2025-08-18T07:50:39.090854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def batch_to_device(batch, device):\n    return {k:batch[k].to(device, non_blocking=True) for k in batch.keys() if k not in []}\n\n\ndef train_epoch(loader, model, optimizer, scheduler, scaler, device, cfg, epoch):\n    model.train()\n    model.zero_grad()\n    loss_output = []\n    grad_norm_l = []\n    accumulate = cfg.accumulate\n\n    for batch in iter(loader):\n        input_dict = batch_to_device(batch, device)\n        with autocast(device_type=\"cuda\", enabled=True):\n            targets = input_dict['targets']\n            features = input_dict['features']\n            logits = model(features)\n            loss = cfg.loss_func(logits, targets)\n\n        loss_output.append(loss.detach().cpu().item())\n        scaler.scale(loss / cfg.accumulate).backward() \n        accumulate -= 1\n        \n        if accumulate == 0:\n            current_scale = scaler.get_scale()\n            scaler.unscale_(optimizer)\n            nn.utils.clip_grad_value_(model.parameters(), cfg.grad_value)\n            scaler.step(optimizer)     \n            scaler.update()\n            optimizer.zero_grad()\n            accumulate = cfg.accumulate\n\n        scheduler.step()\n\n\ndef valid_epoch(loader, model, device, cfg):\n    model.eval()\n    model.zero_grad()\n    preds_output = []\n    targets_output = []\n    with torch.no_grad():\n        for batch in iter(loader):      \n            input_dict = batch_to_device(batch, device)\n            features = input_dict['features']\n            with autocast(device_type=\"cuda\", enabled=False):\n                preds = model(features)\n                    \n            preds_output.append(preds.detach().cpu())\n            targets = batch['targets']\n            targets_output.append(targets)\n\n    preds_output = torch.cat(preds_output, dim=0)\n    targets_output = torch.cat(targets_output, dim=0)\n    return preds_output.numpy(), targets_output.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:50:39.092286Z","iopub.execute_input":"2025-08-18T07:50:39.092566Z","iopub.status.idle":"2025-08-18T07:50:39.111986Z","shell.execute_reply.started":"2025-08-18T07:50:39.092539Z","shell.execute_reply":"2025-08-18T07:50:39.111142Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CMI training part ","metadata":{}},{"cell_type":"code","source":"cfg.loss_func = nn.CrossEntropyLoss()\n\nrecord_oof = []\nrecord_pred = []\nfeature_dim = len(cfg.imc_index) + len(cfg.gen_index)\n\nfor fold in [0, 1, 2, 3]:\n\n    print(f\"*************************** Fold {fold} ***************************\")\n    train_fold = demo[demo.fold != fold].reset_index(drop=True)\n    valid_fold = demo[demo.fold == fold].reset_index(drop=True)\n    \n    train_size = len(train_fold)\n    valid_size = len(valid_fold)\n    print(f\"train size: {train_size}, valid size: {valid_size}\")\n    df_oof = valid_fold[['sequence_id', 'gesture']].copy()\n    df_oof['fold'] = fold\n    df_pred = valid_fold[['sequence_id']].copy()\n    df_pred['fold'] = fold\n    \n    feat_mean = np.load(f\"/kaggle/working/all_features_mean_fold{fold}.npy\")\n    feat_std = np.load(f\"/kaggle/working/all_features_std_fold{fold}.npy\")\n    \n    valid_ds = CMIDataset(valid_fold, cfg, feat_mean, feat_std, cfg.imc_index)\n    valid_dataloader = get_dataloader(valid_ds, cfg, istrain=False)\n\n    model = CNN1D(cfg, feature_dim).to(cfg.device)\n    optimizer = get_optimizer(model, cfg)\n    scaler = GradScaler()\n    scheduler = None\n    result = None\n    best_score = -1\n    for epoch in tqdm(range(cfg.num_epochs), total=cfg.num_epochs):\n        train_ds = CMIDataset(train_fold, cfg, feat_mean, feat_std, cfg.imc_index)\n        train_dataloader = get_dataloader(train_ds, cfg, istrain=True)\n        if scheduler is None:\n            scheduler = lr_scheduler.OneCycleLR(\n                optimizer,\n                max_lr=cfg.lr,\n                epochs=cfg.num_epochs,\n                steps_per_epoch=len(train_dataloader),\n                pct_start=0.07,\n                anneal_strategy=\"cos\",\n                final_div_factor=0.02,\n            )\n        \n        train_epoch(train_dataloader, model, optimizer, scheduler, scaler, cfg.device, cfg, epoch)\n        preds, targets = valid_epoch(valid_dataloader, model, cfg.device, cfg)\n        # from argmax to string label\n        p_index = preds.argmax(1)\n        df_pred['gesture'] = cfg.targets[p_index]\n    \n        result = score(\n            df_oof[['sequence_id', 'gesture']], \n            df_pred[['sequence_id', 'gesture']],\n            row_id_column_name='sequence_id'\n        )\n    \n        if result > best_score:\n            print(f\"Epoch {epoch+1} [{result:.4f}]\")\n            save_path = f'cmi_model_fold{fold}.pt'\n            checkpoint = {\n                'model' : model.state_dict(),\n                'fold' : fold,\n                'best_score': result\n            }\n            torch.save(checkpoint, save_path)\n            best_score = result\n\n    print()\n    record_oof.append(df_oof)\n    record_pred.append(df_pred)\n    \n    del model, optimizer, scheduler, scaler, train_dataloader, valid_dataloader\n    gc.collect()\n    torch.cuda.empty_cache()\n\nrecord_oof = pd.concat(record_oof, ignore_index=True)\nrecord_pred = pd.concat(record_pred, ignore_index=True)\nrecord_oof.to_csv(\"cmi_oof.csv\", index=False)\nrecord_pred.to_csv(\"cmi_pred.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T07:50:39.112893Z","iopub.execute_input":"2025-08-18T07:50:39.113135Z","iopub.status.idle":"2025-08-18T08:00:58.986906Z","shell.execute_reply.started":"2025-08-18T07:50:39.113110Z","shell.execute_reply":"2025-08-18T08:00:58.986033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score(\n    record_oof[['sequence_id', 'gesture']], \n    record_pred[['sequence_id', 'gesture']],\n    row_id_column_name='sequence_id'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T08:00:58.990305Z","iopub.execute_input":"2025-08-18T08:00:58.990597Z","iopub.status.idle":"2025-08-18T08:00:59.068072Z","shell.execute_reply.started":"2025-08-18T08:00:58.990575Z","shell.execute_reply":"2025-08-18T08:00:59.067351Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CMI+THM training part ","metadata":{}},{"cell_type":"code","source":"cfg.loss_func = nn.CrossEntropyLoss()\n\nrecord_oof = []\nrecord_pred = []\nfeature_dim = len(cfg.all_index) + len(cfg.gen_index)\n\nfor fold in [0, 1, 2, 3]:\n\n    print(f\"*************************** Fold {fold} ***************************\")\n    train_fold = demo[demo.fold != fold].reset_index(drop=True)\n    valid_fold = demo[demo.fold == fold].reset_index(drop=True)\n    \n    train_size = len(train_fold)\n    valid_size = len(valid_fold)\n    print(f\"train size: {train_size}, valid size: {valid_size}\")\n    df_oof = valid_fold[['sequence_id', 'gesture']].copy()\n    df_oof['fold'] = fold\n    df_pred = valid_fold[['sequence_id']].copy()\n    df_pred['fold'] = fold\n    \n    feat_mean = np.load(f\"/kaggle/working/all_features_mean_fold{fold}.npy\")\n    feat_std = np.load(f\"/kaggle/working/all_features_std_fold{fold}.npy\")\n    \n    valid_ds = CMIDataset(valid_fold, cfg, feat_mean, feat_std, cfg.all_index)\n    valid_dataloader = get_dataloader(valid_ds, cfg, istrain=False)\n\n    model = CNN1D(cfg, feature_dim).to(cfg.device)\n    optimizer = get_optimizer(model, cfg)\n    scaler = GradScaler()\n    scheduler = None\n    result = None\n    best_score = -1\n    for epoch in tqdm(range(cfg.num_epochs), total=cfg.num_epochs):\n        train_ds = CMIDataset(train_fold, cfg, feat_mean, feat_std, cfg.all_index)\n        train_dataloader = get_dataloader(train_ds, cfg, istrain=True)\n        if scheduler is None:\n            scheduler = lr_scheduler.OneCycleLR(\n                optimizer,\n                max_lr=cfg.lr,\n                epochs=cfg.num_epochs,\n                steps_per_epoch=len(train_dataloader),\n                pct_start=0.07,\n                anneal_strategy=\"cos\",\n                final_div_factor=0.02,\n            )\n        \n        train_epoch(train_dataloader, model, optimizer, scheduler, scaler, cfg.device, cfg, epoch)\n        preds, targets = valid_epoch(valid_dataloader, model, cfg.device, cfg)\n        # from argmax to string label\n        p_index = preds.argmax(1)\n        df_pred['gesture'] = cfg.targets[p_index]\n    \n        result = score(\n            df_oof[['sequence_id', 'gesture']], \n            df_pred[['sequence_id', 'gesture']],\n            row_id_column_name='sequence_id'\n        )\n    \n        if result > best_score:\n            print(f\"Epoch {epoch+1} [{result:.4f}]\")\n            save_path = f'cmithm_model_fold{fold}.pt'\n            checkpoint = {\n                'model' : model.state_dict(),\n                'fold' : fold,\n                'best_score': result\n            }\n            torch.save(checkpoint, save_path)\n            best_score = result\n\n    print()\n    record_oof.append(df_oof)\n    record_pred.append(df_pred)\n    \n    del model, optimizer, scheduler, scaler, train_dataloader, valid_dataloader\n    gc.collect()\n    torch.cuda.empty_cache()\n\nrecord_oof = pd.concat(record_oof, ignore_index=True)\nrecord_pred = pd.concat(record_pred, ignore_index=True)\nrecord_oof.to_csv(\"cmithm_oof.csv\", index=False)\nrecord_pred.to_csv(\"cmithm_pred.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T08:00:59.068784Z","iopub.execute_input":"2025-08-18T08:00:59.069008Z","iopub.status.idle":"2025-08-18T08:11:21.979028Z","shell.execute_reply.started":"2025-08-18T08:00:59.068991Z","shell.execute_reply":"2025-08-18T08:11:21.978381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score(\n    record_oof[['sequence_id', 'gesture']], \n    record_pred[['sequence_id', 'gesture']],\n    row_id_column_name='sequence_id'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T08:11:21.979884Z","iopub.execute_input":"2025-08-18T08:11:21.980161Z","iopub.status.idle":"2025-08-18T08:11:22.059644Z","shell.execute_reply.started":"2025-08-18T08:11:21.980131Z","shell.execute_reply":"2025-08-18T08:11:22.058965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(test_df, cfg, feature_index, model_path, mean_path, std_path):\n    \"\"\"\n    Run inference cho tập test.\n\n    Args:\n        test_df (pd.DataFrame): DataFrame chứa ['subject','sequence_id','seq_length','f_name'].\n        cfg (SimpleNamespace): config chứa tham số huấn luyện.\n        feature_index (list): danh sách cột đặc trưng (vd: cfg.imc_index hoặc cfg.all_index).\n        model_path (str): đường dẫn file checkpoint đã train (vd: 'cmithm_model_fold0.pt').\n        mean_path (str): đường dẫn file mean (vd: 'all_features_mean_fold0.npy').\n        std_path (str): đường dẫn file std (vd: 'all_features_std_fold0.npy').\n\n    Returns:\n        pd.DataFrame: ['sequence_id','gesture'] dự đoán.\n    \"\"\"\n    # load normalization\n    feat_mean = np.load(mean_path)\n    feat_std = np.load(std_path)\n\n    # tạo dataset và dataloader\n    test_ds = CMIDataset(test_df, cfg, feat_mean, feat_std, feature_index)\n    test_loader = get_dataloader(test_ds, cfg, istrain=False)\n\n    # định nghĩa model và load checkpoint\n    feature_dim = len(feature_index) + len(cfg.gen_index)\n    model = CNN1D(cfg, feature_dim).to(cfg.device)\n    checkpoint = torch.load(model_path, map_location=cfg.device)\n    model.load_state_dict(checkpoint['model'])\n    model.eval()\n\n    preds_output = []\n    with torch.no_grad():\n        for batch in iter(test_loader):\n            input_dict = batch_to_device(batch, cfg.device)\n            features = input_dict['features']\n            preds = model(features)\n            preds_output.append(preds.detach().cpu())\n\n    # ghép kết quả\n    preds_output = torch.cat(preds_output, dim=0)\n    pred_index = preds_output.argmax(1).numpy()\n    pred_labels = cfg.targets[pred_index]\n\n    df_pred = test_df[['sequence_id']].copy()\n    df_pred['gesture'] = pred_labels\n    return df_pred\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T08:11:22.060432Z","iopub.execute_input":"2025-08-18T08:11:22.060708Z","iopub.status.idle":"2025-08-18T08:11:22.068936Z","shell.execute_reply.started":"2025-08-18T08:11:22.060662Z","shell.execute_reply":"2025-08-18T08:11:22.067977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_pred.to_parquet(\"submission.parquet\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T08:11:22.070046Z","iopub.execute_input":"2025-08-18T08:11:22.070367Z","iopub.status.idle":"2025-08-18T08:11:22.187085Z","shell.execute_reply.started":"2025-08-18T08:11:22.070340Z","shell.execute_reply":"2025-08-18T08:11:22.186309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== FIX: KAGGLE INFERENCE (single-fold, safe load) ====\nimport os\nimport polars as pl\nimport numpy as np\nimport torch\n\n# ----- preload artifacts only once -----\n_LOADED = False\n_MODEL = None\n_FEAT_MEAN = None\n_FEAT_STD = None\n_FEATURE_INDEX = None\n_FEATURE_DIM = None\n\ndef _init_artifacts():\n    global _LOADED, _MODEL, _FEAT_MEAN, _FEAT_STD, _FEATURE_INDEX, _FEATURE_DIM\n    if _LOADED:\n        return\n\n    # chọn feature index bạn dùng khi train\n    _FEATURE_INDEX = cfg.all_index                   # hoặc cfg.imc_index\n    _FEATURE_DIM = len(_FEATURE_INDEX) + len(getattr(cfg, \"gen_index\", []))\n\n    # đường dẫn checkpoint & stats (đặt đúng nơi bạn lưu)\n    ckpt_path = \"/kaggle/working/cmithm_model_fold0.pt\"\n    mean_path = \"/kaggle/working/all_features_mean_fold0.npy\"\n    std_path  = \"/kaggle/working/all_features_std_fold0.npy\"\n\n    # nạp mean/std\n    _FEAT_MEAN = np.load(mean_path)\n    _FEAT_STD  = np.load(std_path)\n\n    # tạo model & nạp trọng số\n    _MODEL = CNN1D(cfg, _FEATURE_DIM).to(cfg.device)\n    try:\n        # PyTorch 2.6: cần tắt weights_only để load các object cũ\n        ckpt = torch.load(ckpt_path, map_location=cfg.device, weights_only=False)\n    except TypeError:\n        # phòng trường hợp phiên bản torch không có tham số weights_only\n        ckpt = torch.load(ckpt_path, map_location=cfg.device)\n\n    # hỗ trợ cả 2 kiểu lưu: dict {'model': state_dict} hoặc state_dict thuần\n    state = ckpt['model'] if (isinstance(ckpt, dict) and 'model' in ckpt) else ckpt\n    _MODEL.load_state_dict(state, strict=True)\n    _MODEL.eval()\n\n    _LOADED = True\n\n\ndef _pad_or_trim(x: np.ndarray, target_len: int) -> np.ndarray:\n    \"\"\"pad/trim theo chiều time (axis=0).\"\"\"\n    T, F = x.shape\n    if T > target_len:\n        return x[:target_len]\n    if T < target_len:\n        out = np.zeros((target_len, F), dtype=x.dtype)\n        out[:T] = x\n        return out\n    return x\n\n\ndef _build_input_from_polars(sequence: pl.DataFrame) -> np.ndarray:\n    \"\"\"Lấy đúng thứ tự feature, chuẩn hóa, tạo đặc trưng phụ nếu bạn có.\"\"\"\n    # lấy đúng cột theo thứ tự đã train\n    x = sequence.select(_FEATURE_INDEX).to_numpy().astype(np.float32)\n    # normalize\n    x = (x - _FEAT_MEAN) / (_FEAT_STD + 1e-8)\n    # pad/trim\n    x = _pad_or_trim(x, cfg.max_length)\n\n    # nếu bạn có sinh thêm đặc trưng theo thời gian, nối vào đây.\n    # ví dụ: x = np.concatenate([x, gen_features(x)], axis=1)\n    return x\n\n\n@torch.no_grad()\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    \"\"\"\n    Hàm bắt buộc cho Kaggle: nhận 1 sequence (polars) + demographics, trả về tên gesture.\n    \"\"\"\n    _init_artifacts()  # đảm bảo đã nạp model & stats\n\n    x = _build_input_from_polars(sequence)                 # (T, F)\n    x = torch.from_numpy(x).unsqueeze(0).to(cfg.device)    # (1, T, F)\n\n    logits = _MODEL(x)                                     # (1, C)\n    pred_idx = int(torch.argmax(logits, dim=1).item())\n    return cfg.targets[pred_idx]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T08:19:44.945112Z","iopub.execute_input":"2025-08-18T08:19:44.945927Z","iopub.status.idle":"2025-08-18T08:19:44.957290Z","shell.execute_reply.started":"2025-08-18T08:19:44.945901Z","shell.execute_reply":"2025-08-18T08:19:44.956302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}