{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Tổng quan","metadata":{}},{"cell_type":"markdown","source":"Mục tiêu là khám phá các thuật toán học máy giám sát để phát hiện hành vi dựa trên dữ liệu cảm biến. Chúng ta sẽ xử lý các vấn đề như giá trị thiếu, xây dựng đường ống (pipelines), tạo ra sơ đồ phụ thuộc, và áp dụng mã hóa một lần.\n\nChúng ta sẽ sử dụng dữ liệu từ các cảm biến để dự đoán hành vi của các đối tượng trong môi trường. Các thuật toán học máy sẽ giúp phân loại hành vi dựa trên các đặc điểm đo lường được từ dữ liệu cảm biến. Mục tiêu cuối cùng là xây dựng một mô hình có khả năng dự đoán chính xác hành vi dựa trên các tính năng cảm biến được cung cấp.","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Tải dữ liệu và thư viện","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings\nfrom matplotlib.ticker import MaxNLocator\nfrom mpl_toolkits.mplot3d import Axes3D\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nfrom sklearn.metrics import accuracy_score, f1_score\nimport joblib\nfrom scipy.spatial.transform import Rotation as R","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:47:17.814157Z","iopub.execute_input":"2025-08-21T14:47:17.814489Z","iopub.status.idle":"2025-08-21T14:47:19.801406Z","shell.execute_reply.started":"2025-08-21T14:47:17.814465Z","shell.execute_reply":"2025-08-21T14:47:19.800771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tải tập dữ liệu\n#Dữ liệu cảm biến cho tập huấn luyện (Train Sensor Data): \ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\n#Thông tin nhân khẩu học của đối tượng trong tập huấn luyện (giới tính, tuổi, ...): \ntrain_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\n\n#Dữ liệu cảm biến cho tập kiểm tra:\ntest_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\n\n# Thông tin nhân khẩu học của đối tượng trong tập kiểm tra:\ntest_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:47:19.802566Z","iopub.execute_input":"2025-08-21T14:47:19.802929Z","iopub.status.idle":"2025-08-21T14:47:55.194825Z","shell.execute_reply.started":"2025-08-21T14:47:19.802909Z","shell.execute_reply":"2025-08-21T14:47:55.194272Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.1 Khám phá dữ liệu","metadata":{}},{"cell_type":"code","source":"datasets = {\n    \"Train Data\": train_df,\n    \"Train Demographics\": train_dem_df,\n    \"Test Data\": test_df,\n    \"Test Demographics\": test_dem_df,\n}\n\nfor name, df in datasets.items():\n    print(f\"Missing values in {name}:\")\n    missing = df.isnull().sum()\n    missing = missing[missing > 0]\n    if not missing.empty:\n        print(missing)\n    else:\n        print(\"  ✅ No missing values.\")\n    print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:47:55.195628Z","iopub.execute_input":"2025-08-21T14:47:55.195895Z","iopub.status.idle":"2025-08-21T14:47:55.952492Z","shell.execute_reply.started":"2025-08-21T14:47:55.195872Z","shell.execute_reply":"2025-08-21T14:47:55.951793Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.5 kiểm tra giá trị trùng lặp","metadata":{}},{"cell_type":"code","source":"#Kiểm tra trùng lặp (Duplicate Rows): .duplicated(): Trả về Series boolean, dòng nào trùng lặp sẽ là True.\n#                                     .sum(): Đếm tổng số dòng trùng lặp.\n#Kiểm tra xem dữ liệu có bản ghi nào bị trùng không — quan trọng cho việc làm sạch dữ liệu.\n\n# Đếm các hàng trùng lặp trong train_df\ntrain_duplicates = train_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong  test_df\ntest_duplicates = test_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong train_dem_df (optional)\ntrain_dem_duplicates = train_dem_df.duplicated().sum()\n# Đếm các hàng trùng lặp trong test_dem_df (optional)\ntest_dem_duplicates = test_dem_df.duplicated().sum()\n\n# In số lượng dòng trùng:\nprint(f\"Number of duplicate rows in train_df: {train_duplicates}\")\nprint(f\"Number of duplicate rows in test_df: {test_duplicates}\")\nprint(f\"Number of duplicate rows in train_dem_df: {train_dem_duplicates}\")\nprint(f\"Number of duplicate rows in test_dem_df: {test_dem_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:47:55.954003Z","iopub.execute_input":"2025-08-21T14:47:55.954289Z","iopub.status.idle":"2025-08-21T14:48:00.134280Z","shell.execute_reply.started":"2025-08-21T14:47:55.954270Z","shell.execute_reply":"2025-08-21T14:48:00.133456Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"markdown","source":"#### UTILS/HELPERS","metadata":{}},{"cell_type":"code","source":"def null_percent(df):\n    per=((df.isnull().sum()/len(df))*100).round(2)\n    return per[per>0]\n\nprint(\"Nan Values in Train data\")\nprint(null_percent(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:00.134934Z","iopub.execute_input":"2025-08-21T14:48:00.135164Z","iopub.status.idle":"2025-08-21T14:48:00.888907Z","shell.execute_reply.started":"2025-08-21T14:48:00.135147Z","shell.execute_reply":"2025-08-21T14:48:00.888122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Các cột có tỷ lệ NaN cao nhất là các cột có tên bắt đầu với tof_5_v (5.24% NaN).\n\nCác cột khác như rot_x, rot_w có tỷ lệ NaN thấp hơn nhiều (0.64%).","metadata":{}},{"cell_type":"markdown","source":"## 2.2 Quan sát hoặc xu hướng ban đầu","metadata":{}},{"cell_type":"code","source":"# --- 2.4 Hợp nhất dữ liệu nhân khẩu học và cảm biến ---\nmerged_train = pd.merge(train_df, train_dem_df, on=\"subject\", how=\"left\")\nmerged_test = pd.merge(test_df, test_dem_df, on=\"subject\", how=\"left\")\n\nprint(\"Merged Train Shape:\", merged_train.shape)\nprint(\"Merged Test Shape:\", merged_test.shape)\ndisplay(merged_train.head(2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:00.889820Z","iopub.execute_input":"2025-08-21T14:48:00.890110Z","iopub.status.idle":"2025-08-21T14:48:01.536477Z","shell.execute_reply.started":"2025-08-21T14:48:00.890085Z","shell.execute_reply":"2025-08-21T14:48:01.535867Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Merged Train Shape:** (574945, 348) - Sau khi hợp nhất, merged_train có 574,945 dòng và 348 cột, tức là dữ liệu từ train_df (574,945 dòng và 341 cột) đã được kết hợp với 7 cột từ train_dem_df (vì train_dem_df có 8 cột, nhưng 1 cột là subject đã được sử dụng làm khóa hợp nhất).\n\n**Merged Test Shape:** (107, 343) - Tương tự, merged_test có 107 dòng và 343 cột, kết quả là hợp nhất giữa test_df (107 dòng và 336 cột) với test_dem_df (8 cột, với 1 cột subject là khóa hợp nhất).","metadata":{}},{"cell_type":"code","source":"# --- Thống kê mô tả toàn bộ merged_train ---\nmerged_train.describe().T\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:01.537210Z","iopub.execute_input":"2025-08-21T14:48:01.537469Z","iopub.status.idle":"2025-08-21T14:48:08.772443Z","shell.execute_reply.started":"2025-08-21T14:48:01.537449Z","shell.execute_reply":"2025-08-21T14:48:08.771704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra thiếu giá trị & thống kê cảm biến: \n#Danh sách các cột không thuộc cảm biến (có thể là ID, thông tin khác). \n\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nsensor_cols = [col for col in train_df.columns if not col.startswith(excluded_prefixes)]\n\n# Sensor Data Summary for TRAIN\n#isnull().sum(): Đếm số giá trị bị thiếu.\nmissing_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    '[TRAIN] Missing Count': train_df[sensor_cols].isnull().sum().values,\n    '[TRAIN] Missing %': (train_df[sensor_cols].isnull().sum().values / len(train_df)) * 100\n})\n\n#nunique(): Đếm số lượng giá trị duy nhất.\nunique_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Unique Values [TRAIN]': train_df[sensor_cols].nunique().values\n})\n\n#dtypes: Lấy kiểu dữ liệu của từng cột.\ndtypes_sensor = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Data Type': train_df[sensor_cols].dtypes.values\n})\n\n# Merge all summaries (NO test set)\n#merge: Gộp các bảng thống kê thành bảng duy nhất theo Feature.\nsensor_summary = missing_sensor_train \\\n    .merge(unique_sensor_train, on='Feature', how='left') \\\n    .merge(dtypes_sensor, on='Feature', how='left')\n\n# Display styled DataFrame (mask NaNs just for styling)\n#fillna(0): Điền giá trị thiếu bằng 0 (cho đẹp mắt khi hiển thị).\n#.style.background_gradient: Tô màu nền theo giá trị giúp dễ nhìn.\nstyled_df = sensor_summary.fillna(0)\nstyled_df.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:08.773421Z","iopub.execute_input":"2025-08-21T14:48:08.773683Z","iopub.status.idle":"2025-08-21T14:48:09.681693Z","shell.execute_reply.started":"2025-08-21T14:48:08.773661Z","shell.execute_reply":"2025-08-21T14:48:09.681035Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Không có giá trị thiếu: Tất cả các cột trong bảng thống kê đều không có giá trị thiếu (Missing Count = 0), cho thấy dữ liệu của bạn rất đầy đủ.\n\n- Sự đa dạng trong dữ liệu: Các cột như row_id, sequence_id, và subject có số lượng giá trị duy nhất khá lớn, trong khi các cột như sequence_type, phase, và behavior có ít giá trị duy nhất hơn, cho thấy chúng có thể là các thông tin phân loại.\n\n- Kiểu dữ liệu: Hầu hết các cột đều có kiểu dữ liệu object, ngoại trừ sequence_counter có kiểu int64, điều này cho thấy hầu hết các cột chứa thông tin phân loại (string), chỉ có một cột chứa giá trị số nguyên.","metadata":{}},{"cell_type":"code","source":"#Thống kê tương tự cho nhân khẩu học: Tương tự như phần thống kê cảm biến nhưng áp dụng cho dữ liệu nhân khẩu học.\n\n# Cột nhân khẩu học (không loại trừ)\ndem_cols = train_dem_df.columns\n\n# Giá trị bị thiếu trong dữ liệu nhân khẩu học của train \nmissing_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    '[TRAIN DEMO] Missing Count': train_dem_df[dem_cols].isnull().sum().values,\n    '[TRAIN DEMO] Missing %': (train_dem_df[dem_cols].isnull().sum().values / len(train_dem_df)) * 100\n})\n\n# Giá trị duy nhất được tính trong dữ liệu nhân khẩu học của train \nunique_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    'Unique Values [TRAIN DEMO]': train_dem_df[dem_cols].nunique().values\n})\n\n# Data types\ndtypes_demo = pd.DataFrame({\n    'Feature': dem_cols,\n    'Data Type': train_dem_df[dem_cols].dtypes.values\n})\n\n# Tóm tắt hợp nhất (chỉ dành cho đào tạo)\ndemo_summary = (\n    missing_demo_train\n    .merge(unique_demo_train, on='Feature', how='left')\n    .merge(dtypes_demo, on='Feature', how='left')\n)\n\n# Hiển thị tóm tắt theo phong cách\ndemo_summary.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:09.682390Z","iopub.execute_input":"2025-08-21T14:48:09.682636Z","iopub.status.idle":"2025-08-21T14:48:09.704208Z","shell.execute_reply.started":"2025-08-21T14:48:09.682621Z","shell.execute_reply":"2025-08-21T14:48:09.703607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Không có giá trị thiếu: Các cột nhân khẩu học đều không có giá trị thiếu, cho thấy dữ liệu đầy đủ và sẵn sàng để phân tích.\n\n- Sự đa dạng trong dữ liệu: Các cột như subject, age, và height_cm có sự đa dạng về giá trị duy nhất, trong khi các cột như sex, handedness có ít giá trị duy nhất hơn, cho thấy đây là các cột phân loại.\n\n- Kiểu dữ liệu: Các cột phần lớn có kiểu dữ liệu int64 hoặc float64, cho thấy dữ liệu chủ yếu là số nguyên hoặc số thực. Các cột như subject có kiểu dữ liệu object, có thể là các chuỗi ký tự.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# 1) Sao chép đào tạo và kiểm tra để chúng ta không sửa đổi DataFrame gốc\ntrain_temp = train_df.copy()\ntest_temp  = test_df.copy()\n\n# 2) GIA TỐC KẾ: tính toán độ lớn tại mỗi dấu thời gian\ntrain_temp['acc_mag'] = np.sqrt(\n    train_temp['acc_x']**2 + train_temp['acc_y']**2 + train_temp['acc_z']**2\n)\ntest_temp['acc_mag'] = np.sqrt(\n    test_temp['acc_x']**2 + test_temp['acc_y']**2 + test_temp['acc_z']**2\n)\n\n# 3) ROTATION: tính toán “góc quay” từ thành phần quaternion w\n# (Lưu ý: rot_w nằm trong [-1,1], do đó arccos hợp lệ. Chúng tôi bỏ qua NaN nếu có.)\ntrain_temp['rot_angle'] = 2 * np.arccos(train_temp['rot_w'].clip(-1,1))\ntest_temp['rot_angle']  = 2 * np.arccos(test_temp['rot_w'].clip(-1,1))\n\n# 4) Nhóm theo sequence_id và tổng hợp các tóm tắt gia tốc kế\nacc_agg_funcs = {\n    'acc_mag': ['mean', 'std', 'max']\n}\ntrain_acc_summary = train_temp.groupby('sequence_id').agg(acc_agg_funcs)\ntest_acc_summary  = test_temp.groupby('sequence_id').agg(acc_agg_funcs)\n\n# Làm phẳng cột MultiIndex\ntrain_acc_summary.columns = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\ntest_acc_summary.columns  = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\n\n# 5) Nhóm theo sequence_id và tổng hợp tóm tắt vòng quay\nrot_agg_funcs = {\n    'rot_angle': ['mean', 'std', 'max']\n}\ntrain_rot_summary = train_temp.groupby('sequence_id').agg(rot_agg_funcs)\ntest_rot_summary  = test_temp.groupby('sequence_id').agg(rot_agg_funcs)\n\ntrain_rot_summary.columns = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\ntest_rot_summary.columns  = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\n\n# 6) NHIỆT ĐỘ: năm cảm biến thm_1 … thm_5\nthm_cols = [f\"thm_{i}\" for i in range(1, 6)]\n\n# Xác định hàm tổng hợp: trung bình + độ lệch chuẩn\nthm_agg_funcs = {col: ['mean', 'std'] for col in thm_cols}\n\ntrain_thm_summary = train_temp.groupby('sequence_id').agg(thm_agg_funcs)\ntest_thm_summary  = test_temp.groupby('sequence_id').agg(thm_agg_funcs)\n\n# Làm phẳng các cột MultiIndex\nflattened_thm_cols = []\nfor sensor in thm_cols:\n    for stat in ['mean','std']:\n        flattened_thm_cols.append(f\"{sensor}_{stat}\")\n\ntrain_thm_summary.columns = flattened_thm_cols\ntest_thm_summary.columns  = flattened_thm_cols\n\n# 7) THỜI GIAN BAY: mỗi cảm biến i có 64 cột pixel: tof_i_v0 … tof_i_v63\n# Chúng ta sẽ tạo một “tof_i_mean_at_ts” cho mỗi dấu thời gian, sau đó tổng hợp theo từng chuỗi.\n\ndef compute_tof_sequence_summary(df):\n    # Khởi tạo một dict để giữ DataFrames theo từng chuỗi\n    seq_summaries = {}\n\n    for i in range(1, 6):\n        # Xây dựng danh sách các cột cho cảm biến i\n        tof_cols = [f\"tof_{i}_v{pix}\" for pix in range(64)]\n        # Thay thế -1 bằng NaN để chúng không làm lệch giá trị trung bình; chuyển đổi sang float\n        ts_grid = df[tof_cols].replace(-1, np.nan).astype(float)\n        # Tính toán “trung bình trên tất cả 64 pixel” cho mỗi dấu thời gian\n        df[f\"tof_{i}_mean_at_ts\"] = ts_grid.mean(axis=1)\n    \n   # Bây giờ, nhóm theo id chuỗi và tính giá trị trung bình và độ lệch chuẩn của các giá trị trung bình đó\n    agg_dict = {f\"tof_{i}_mean_at_ts\": ['mean','std'] for i in range(1, 6)}\n    summary = df.groupby('sequence_id').agg(agg_dict)\n    # Làm phẳng các cột MultiIndex\n    flat_cols = [f\"tof_{i}_{stat}\" for i in range(1, 6) for stat in ['mean','std']]\n    summary.columns = flat_cols\n    return summary\n\ntrain_tof_summary = compute_tof_sequence_summary(train_temp)\ntest_tof_summary  = compute_tof_sequence_summary(test_temp)\n\n# 8) Hợp nhất các tóm tắt accel, rotation, thm, tof (trên sequence_id)\ntrain_sensor_summary = (\n    train_acc_summary\n    .join(train_rot_summary, how='outer')\n    .join(train_thm_summary, how='outer')\n    .join(train_tof_summary, how='outer')\n)\n\ntest_sensor_summary = (\n    test_acc_summary\n    .join(test_rot_summary, how='outer')\n    .join(test_thm_summary, how='outer')\n    .join(test_tof_summary, how='outer')\n)\n\n# 9) Thêm cột “Dataset” để chúng ta có thể thực hiện box+hist song song\ntrain_sensor_summary['Dataset'] = 'Train'\ntest_sensor_summary['Dataset']  = 'Test'\n\n# 10) Ghép nối thành một DataFrame để vẽ đồ thị\ncombined_sensor_summary = pd.concat(\n    [train_sensor_summary, test_sensor_summary],\n    axis=0\n).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:09.706542Z","iopub.execute_input":"2025-08-21T14:48:09.706935Z","iopub.status.idle":"2025-08-21T14:48:15.044903Z","shell.execute_reply.started":"2025-08-21T14:48:09.706918Z","shell.execute_reply":"2025-08-21T14:48:15.044393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# (1) Sáp nhập train_demographics vào train_df nếu chưa thực hiện\ntrain_df = train_df.merge(\n    train_dem_df,\n    on=\"subject\",\n    how=\"left\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.045515Z","iopub.execute_input":"2025-08-21T14:48:15.045695Z","iopub.status.idle":"2025-08-21T14:48:15.656667Z","shell.execute_reply.started":"2025-08-21T14:48:15.045680Z","shell.execute_reply":"2025-08-21T14:48:15.656112Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Xử lý missing values","metadata":{}},{"cell_type":"markdown","source":"### 1. IMU – Gia tốc/quaternion (acc_*, rot_w/x/y/z)","metadata":{}},{"cell_type":"code","source":"def fix_imu(df):\n    imu_cols = [c for c in df.columns if c.startswith((\"acc_\", \"rot_\"))]\n    def _seq_fix(g):\n        g[imu_cols] = g[imu_cols].interpolate(limit_direction=\"both\")\n        # renormalize quaternion\n        q = g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]].to_numpy(float)\n        n = (q**2).sum(1)**0.5\n        n[n==0] = 1.0\n        q = q / n[:,None]\n        g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]] = q\n        return g\n    return df.groupby(\"sequence_id\", group_keys=False).apply(_seq_fix)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.657610Z","iopub.execute_input":"2025-08-21T14:48:15.657820Z","iopub.status.idle":"2025-08-21T14:48:15.662555Z","shell.execute_reply.started":"2025-08-21T14:48:15.657804Z","shell.execute_reply":"2025-08-21T14:48:15.661831Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. Thermopile (thm_1..thm_5)","metadata":{}},{"cell_type":"code","source":"def fix_thm(df):\n    thm = [f\"thm_{i}\" for i in range(1,6)]\n    return df.groupby(\"sequence_id\", group_keys=False)[thm].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.663312Z","iopub.execute_input":"2025-08-21T14:48:15.663502Z","iopub.status.idle":"2025-08-21T14:48:15.675969Z","shell.execute_reply.started":"2025-08-21T14:48:15.663487Z","shell.execute_reply":"2025-08-21T14:48:15.675330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. ToF – 5 cảm biến × 64 pixel (tof_i_vj)","metadata":{}},{"cell_type":"code","source":"def fix_tof_and_reduce(df):\n    # 1) sentinel -> NaN + flag\n    tof_pix = [c for c in df.columns if c.startswith(\"tof_\") and \"_v\" in c]\n    df[tof_pix] = df[tof_pix].replace(-1, np.nan)\n    # optional flags per sensor\n    for i in range(1,6):\n        cols = [f\"tof_{i}_v{p}\" for p in range(64) if f\"tof_{i}_v{p}\" in df]\n        df[f\"tof_{i}_missing_frac\"] = df[cols].isna().mean(axis=1)\n        df[f\"tof_{i}_mean\"] = df[cols].mean(axis=1)  # dimension reduction\n    # 2) interpolate the 5 means\n    mean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\n    df[mean_cols] = df.groupby(\"sequence_id\", group_keys=False)[mean_cols].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )\n    # 3) drop heavy pixel columns to save RAM\n    df.drop(columns=tof_pix, inplace=True)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.676717Z","iopub.execute_input":"2025-08-21T14:48:15.676939Z","iopub.status.idle":"2025-08-21T14:48:15.687555Z","shell.execute_reply.started":"2025-08-21T14:48:15.676919Z","shell.execute_reply":"2025-08-21T14:48:15.687012Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 4. Pha/nhãn & metadata","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle_gesture = LabelEncoder().fit(train_df[\"gesture\"].astype(str))\ntrain_df[\"gesture_id\"] = le_gesture.transform(train_df[\"gesture\"].astype(str))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.688232Z","iopub.execute_input":"2025-08-21T14:48:15.688439Z","iopub.status.idle":"2025-08-21T14:48:15.795747Z","shell.execute_reply.started":"2025-08-21T14:48:15.688425Z","shell.execute_reply":"2025-08-21T14:48:15.794983Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5. Pipeline tiền xử lý","metadata":{}},{"cell_type":"code","source":"def minimal_preprocess(df):\n    df = df.copy()\n    # 1) IMU\n    df = fix_imu(df)\n    # 2) Thermopile\n    thm_fixed = fix_thm(df)  # returns only thm cols\n    df[thm_fixed.columns] = thm_fixed\n    # 3) ToF -> means + flags, drop pixels\n    df = fix_tof_and_reduce(df)\n    # 4) Feature cơ bản (ví dụ):\n    df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n    df[\"rot_angle\"] = 2*np.arccos(np.clip(df[\"rot_w\"], -1, 1))\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.796863Z","iopub.execute_input":"2025-08-21T14:48:15.797097Z","iopub.status.idle":"2025-08-21T14:48:15.801257Z","shell.execute_reply.started":"2025-08-21T14:48:15.797079Z","shell.execute_reply":"2025-08-21T14:48:15.800459Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Cấu hình tham số và thư viện cần thiết:","metadata":{}},{"cell_type":"code","source":"train_df['gesture'].unique()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.801918Z","iopub.execute_input":"2025-08-21T14:48:15.802155Z","iopub.status.idle":"2025-08-21T14:48:15.842602Z","shell.execute_reply.started":"2025-08-21T14:48:15.802139Z","shell.execute_reply":"2025-08-21T14:48:15.842105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# GESTURE MAPPING\n# =============================================================================\n\n# Gesture mapping (targets 0-7 are BFRB, 8-17 are non-BFRB)\nGESTURE_MAPPER = {\n    \"Above ear - pull hair\": 0,\n    \"Cheek - pinch skin\": 1,\n    \"Eyebrow - pull hair\": 2,\n    \"Eyelash - pull hair\": 3, \n    \"Forehead - pull hairline\": 4,\n    \"Forehead - scratch\": 5,\n    \"Neck - pinch skin\": 6, \n    \"Neck - scratch\": 7,\n    \n    \"Drink from bottle/cup\": 8,\n    \"Feel around in tray and pull out an object\": 9,\n    \"Glasses on/off\": 10,\n    \"Pinch knee/leg skin\": 11, \n    \"Pull air toward your face\": 12,\n    \"Scratch knee/leg skin\": 13,\n    \"Text on phone\": 14,\n    \"Wave hello\": 15,\n    \"Write name in air\": 16,\n    \"Write name on leg\": 17,\n}\n\nREVERSE_GESTURE_MAPPER = {v: k for k, v in GESTURE_MAPPER.items()}\\\n\n# =============================================================================\n# GROUPED LABELS FOR CONTEST METRIC\n# =============================================================================\n\n# ID các lớp target (BFRB)\ntarget_ids = set(range(0, 8))  # 0-7\n\n# ID các lớp non-target (non-BFRB)\nnon_target_ids = set(range(8, 18))  # 8-17\n\n# Mapping id -> grouped_label\nid2label_grouped = {i: ('non_target' if i in non_target_ids else REVERSE_GESTURE_MAPPER[i])\n                    for i in range(len(GESTURE_MAPPER))}\n\n# Mapping grouped_label -> id\nunique_grouped_labels = sorted(set(id2label_grouped.values()))\nlabel2id_grouped = {g: i for i, g in enumerate(unique_grouped_labels)}\nid2label_grouped_final = {i: g for g, i in label2id_grouped.items()}\n\n# ID lớp non_target trong grouped mapping\nnon_target_id = label2id_grouped['non_target']\ntarget_ids_grouped = {i for i in id2label_grouped_final if i != non_target_id}\n\nprint(\"Target IDs (original mapping):\", target_ids)\nprint(\"Non-target IDs (original mapping):\", non_target_ids)\nprint(\"Grouped label2id:\", label2id_grouped)\n\n# ==== GROUPED LABELS: 8 target + 1 non_target (gộp 10 lớp 8–17) ====\n\n# 1) Lấy danh sách 8 target theo thứ tự ID gốc 0..7\nTARGET_LABELS = [name for name, idx in sorted(GESTURE_MAPPER.items(), key=lambda kv: kv[1]) if idx <= 7]\n\n# 2) Định nghĩa nhãn gộp\nGROUPED_LABELS = TARGET_LABELS + ['non_target']   # giữ nguyên 8 target, + 1 lớp non_target\n\n# 3) Mapping NHÓM (ghi đè mapping cũ để dùng xuyên suốt pipeline)\nlabel2id = {name: i for i, name in enumerate(GROUPED_LABELS)}\nid2label = {i: name for name, i in label2id.items()}\n\nNON_TARGET = 'non_target'\nassert NON_TARGET in label2id, \"Grouped mapping chưa có non_target!\"\nnon_target_id = label2id[NON_TARGET]\ntarget_ids = {i for i in id2label if i != non_target_id}\n\n# 4) Tạo y_train theo NHÃN GỘP từ tên gesture gốc trong train_df\ndef to_grouped_label_from_name(name: str) -> str:\n    return name if name in TARGET_LABELS else 'non_target'\n\ntrain_df['gesture_grouped'] = train_df['gesture'].apply(to_grouped_label_from_name)\ny_train = train_df['gesture_grouped'].map(label2id).values\n\n# (Tuỳ biến) Nếu bạn có X_train (features) đã tạo sẵn:\n# feature_cols = [c for c in X_train.columns if c != 'sequence_id']\n# X_features = X_train[feature_cols]\n\n# 5) In kiểm tra\nprint(\"TARGET_LABELS:\", TARGET_LABELS)\nprint(\"GROUPED_LABELS:\", GROUPED_LABELS)\nprint(\"label2id:\", label2id)\nprint(\"Counts grouped:\", train_df['gesture_grouped'].value_counts().to_dict())\nprint(\"OK: non_target_id =\", non_target_id, \"| target_ids =\", target_ids)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:15.843254Z","iopub.execute_input":"2025-08-21T14:48:15.843464Z","iopub.status.idle":"2025-08-21T14:48:16.021883Z","shell.execute_reply.started":"2025-08-21T14:48:15.843441Z","shell.execute_reply":"2025-08-21T14:48:16.021199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport joblib\nfrom typing import Tuple, List, Optional\nimport warnings\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom lightgbm import LGBMClassifier, log_evaluation, early_stopping\nfrom scipy.spatial.transform import Rotation as R\nimport kaggle_evaluation.cmi_inference_server\n\nwarnings.filterwarnings('ignore')\n\n# =============================================================================\n# CONFIGURATION\n# =============================================================================\n\nclass Config:\n    \"\"\"Central configuration class for training and data parameters\"\"\"\n    \n    # Paths for Kaggle environment\n    TRAIN_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\n    TRAIN_DEMOGRAPHICS_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\"\n    TEST_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\"\n    TEST_DEMOGRAPHICS_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\"\n    \n    # Training parameters\n    SEED = 42\n    N_FOLDS = 5\n    NUM_CLASSES = len(label2id) if 'label2id' in globals() else None\n\n    \n    # Feature columns\n    ACC_COLS = ['acc_x', 'acc_y', 'acc_z']\n    ROT_COLS = ['rot_w', 'rot_x', 'rot_y', 'rot_z']\n    \n    # LightGBM hyperparameters\n    LGBM_PARAMS = {\n        'objective':'multiclass',\n        'num_class': NUM_CLASSES if NUM_CLASSES is not None else 9,\n        'boosting_type':'gbdt',\n        'learning_rate':0.05,\n        'n_estimators':2000,          # dùng early_stopping để dừng sớm\n        'num_leaves':63,\n        'max_depth':-1,\n        'min_data_in_leaf':40,\n        'feature_fraction':0.75,\n        'bagging_fraction':0.75,\n        'bagging_freq':1,\n        'lambda_l1':1.0,\n        'lambda_l2':2.0,\n        'class_weight':'balanced',       # Suppress output verbosity\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:16.022719Z","iopub.execute_input":"2025-08-21T14:48:16.023224Z","iopub.status.idle":"2025-08-21T14:48:20.224691Z","shell.execute_reply.started":"2025-08-21T14:48:16.023206Z","shell.execute_reply":"2025-08-21T14:48:20.224110Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Cấu trúc mô hình và huấn luyện:","metadata":{}},{"cell_type":"code","source":"def check_gpu_availability():\n    \"\"\"Check if GPU is available for LightGBM\"\"\"\n    try:\n        # Try to create a simple LightGBM model with GPU\n        from lightgbm import LGBMClassifier\n        import numpy as np\n        \n        # Create dummy data\n        X_dummy = np.random.rand(100, 10)\n        y_dummy = np.random.randint(0, 2, 100)\n        \n        # Try GPU\n        model = LGBMClassifier(n_estimators=1, device='gpu', verbosity=-1)\n        model.fit(X_dummy, y_dummy)\n        print(\"✓ GPU available for LightGBM\")\n        return 'gpu'\n    except:\n        print(\"⚠️ GPU not available, using CPU for LightGBM\")\n        return 'cpu'\n\n# Initialize DEVICE based on the availability of GPU\nDEVICE = check_gpu_availability()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:20.225392Z","iopub.execute_input":"2025-08-21T14:48:20.225858Z","iopub.status.idle":"2025-08-21T14:48:24.240828Z","shell.execute_reply.started":"2025-08-21T14:48:20.225827Z","shell.execute_reply":"2025-08-21T14:48:24.240185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score\nimport numpy as np\n\ndef contest_score(y_true_ids, y_pred_ids, id2label, non_target_id):\n    # Binary-F1: target vs non_target\n    y_true_bin = (np.array(y_true_ids) != non_target_id).astype(int)\n    y_pred_bin = (np.array(y_pred_ids) != non_target_id).astype(int)\n    f1_binary = f1_score(y_true_bin, y_pred_bin, average='binary', zero_division=0)\n\n    # Macro-F1: gộp tất cả non-target thành một lớp duy nhất\n    f1_macro = f1_score(y_true_ids, y_pred_ids, average='macro', zero_division=0)\n\n    return (f1_binary + f1_macro) / 2.0, f1_binary, f1_macro\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.241856Z","iopub.execute_input":"2025-08-21T14:48:24.242328Z","iopub.status.idle":"2025-08-21T14:48:24.247048Z","shell.execute_reply.started":"2025-08-21T14:48:24.242306Z","shell.execute_reply":"2025-08-21T14:48:24.246384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === Contest score: (Binary-F1 + Macro-F1)/2 ===\nfrom sklearn.metrics import f1_score\nimport numpy as np\n\ndef contest_score(y_true_ids, y_pred_ids, id2label, non_target_id):\n    y_true_bin = (np.array(y_true_ids) != non_target_id).astype(int)\n    y_pred_bin = (np.array(y_pred_ids) != non_target_id).astype(int)\n    f1_binary = f1_score(y_true_bin, y_pred_bin, average='binary', zero_division=0)\n    f1_macro = f1_score(y_true_ids, y_pred_ids, average='macro', zero_division=0)\n    return (f1_binary + f1_macro) / 2.0, f1_binary, f1_macro\n\ndef predict_with_tau(proba, non_target_id, target_ids, tau):\n    y_pred = []\n    for p in proba:\n        best_target = max(target_ids, key=lambda k: p[k])\n        if p[best_target] < tau:\n            y_pred.append(non_target_id)\n        else:\n            y_pred.append(int(np.argmax(p)))\n    return np.array(y_pred, dtype=int)\n\ndef tune_tau_for_cv(valid_y_true, valid_proba, non_target_id, target_ids, id2label):\n    best = (None, -1, None, None)\n    for tau in np.linspace(0.1, 0.9, 17):\n        y_pred = predict_with_tau(valid_proba, non_target_id, target_ids, tau)\n        score, f1_bin, f1_mac = contest_score(valid_y_true, y_pred, id2label, non_target_id)\n        if score > best[1]:\n            best = (tau, score, f1_bin, f1_mac)\n    return best\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.247866Z","iopub.execute_input":"2025-08-21T14:48:24.248115Z","iopub.status.idle":"2025-08-21T14:48:24.265993Z","shell.execute_reply.started":"2025-08-21T14:48:24.248091Z","shell.execute_reply":"2025-08-21T14:48:24.265251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# MODEL BUILDING\n# =============================================================================\n\ndef build_lightgbm_model():\n    \"\"\"Build and return a LightGBM model based on configuration\"\"\"\n    model = LGBMClassifier(**Config.LGBM_PARAMS)\n    return model\n\n# =============================================================================\n# MODEL TRAINING (CV + OOF proba + tau tuning)\n# YÊU CẦU: các biến sau đã có trong scope trước khi gọi:\n# - label2id, id2label: mapping nhãn\n# - non_target_id: int id của lớp 'non_target'\n# - target_ids: set các id lớp target (khác non_target_id)\n# - Config.LGBM_PARAMS, Config.N_FOLDS, Config.SEED\n# - DEVICE: 'cpu' hoặc 'gpu' (tuỳ bạn detect)\n# - competition_metric: HÀM CHẤM ĐIỂM CHUẨN CONTEST (nếu chưa có, dùng contest_score bên dưới)\n# =============================================================================\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score\nimport numpy as np\n\ndef _predict_with_tau(proba, non_target_id, target_ids, tau):\n    \"\"\"\n    proba: ndarray (n_samples, n_classes)\n    Luật: nếu max(p[target]) < tau -> non_target, else -> argmax toàn bộ\n    \"\"\"\n    y_pred = []\n    for p in proba:\n        best_target_id = max(target_ids, key=lambda k: p[k]) if len(target_ids) > 0 else None\n        best_target_p = p[best_target_id] if best_target_id is not None else 0.0\n        if best_target_p < tau:\n            y_pred.append(non_target_id)\n        else:\n            y_pred.append(int(np.argmax(p)))\n    return np.array(y_pred, dtype=int)\n\ndef _tune_tau(y_true_ids, proba, non_target_id, target_ids, id2label):\n    best = (None, -1.0, None, None)  # tau, score, f1_bin, f1_macro\n    for tau in np.linspace(0.1, 0.9, 17):\n        y_pred_ids = _predict_with_tau(proba, non_target_id, target_ids, tau)\n        score, f1_bin, f1_mac = competition_metric(y_true_ids, y_pred_ids)\n        if score > best[1]:\n            best = (float(tau), float(score), float(f1_bin), float(f1_mac))\n    return best  # (tau, score, f1_bin, f1_mac)\n\ndef train_lightgbm_models(X_train, y_train, subjects):\n    \"\"\"\n    Train LightGBM với StratifiedGroupKFold, lưu OOF predict_proba,\n    tune ngưỡng tau theo từng fold, tính OOF score theo tau* (median).\n    Trả về:\n        models: list các model theo fold\n        cv_scores: list contest score từng fold (sau khi áp tau-fold)\n        overall_score: OOF contest score với tau*\n        tau_star: median của các tau theo fold (dùng cho inference)\n        feature_cols: danh sách cột feature (để dùng lại ở predict)\n    \"\"\"\n\n    print(f\"Training LightGBM models with {Config.N_FOLDS}-fold cross-validation...\")\n\n    # 1) Chuẩn bị feature (loại sequence_id nếu có)\n    feature_cols = [c for c in X_train.columns if c != 'sequence_id']\n    X_features = X_train[feature_cols].copy()\n\n    n_classes = len(np.unique(y_train))\n    print(f\"Feature columns: {len(feature_cols)} | Num classes: {n_classes}\")\n    print(\"Starting cross-validation...\")\n\n    # 2) CV setup\n    cv = StratifiedGroupKFold(n_splits=Config.N_FOLDS, shuffle=True, random_state=Config.SEED)\n\n    models = []\n    cv_scores = []\n    taus = []\n\n    # OOF proba để tính điểm tổng thể với một tau*\n    oof_proba = np.zeros((len(X_features), n_classes), dtype=float)\n\n    for fold, (tr_idx, va_idx) in enumerate(cv.split(X_features, y_train, groups=subjects), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_tr, X_va = X_features.iloc[tr_idx], X_features.iloc[va_idx]\n        y_tr, y_va = y_train[tr_idx], y_train[va_idx]\n\n        print(f\"Train size: {len(X_tr)}, Val size: {len(X_va)}\")\n\n        # 3) Khởi tạo model + device\n        lgbm_params = Config.LGBM_PARAMS.copy()\n        lgbm_params['device'] = DEVICE  # ví dụ: 'gpu' nếu bạn đã detect\n        model = LGBMClassifier(**lgbm_params)\n\n        # 4) Train với early_stopping (metric chỉ để dừng sớm)\n        print(f\"Training fold {fold} (Device: {DEVICE})...\")\n        model.fit(\n            X_tr, y_tr,\n            eval_set=[(X_tr, y_tr), (X_va, y_va)],\n            eval_names=['train', 'valid'],\n            eval_metric=['multi_logloss', 'multi_error'],\n            callbacks=[\n                log_evaluation(period=25),\n                early_stopping(stopping_rounds=100, verbose=True)\n            ]\n        )\n        models.append(model)\n\n        # 5) Lưu proba validation\n        proba_va = model.predict_proba(X_va)  # (n_val, n_classes)\n        oof_proba[va_idx] = proba_va\n\n        # 6) Tune tau theo fold\n        tau, score, f1_bin, f1_mac = _tune_tau(y_va, proba_va, non_target_id, target_ids, id2label)\n        taus.append(tau)\n        cv_scores.append(score)\n\n        # 7) In điểm fold với tau tốt nhất của fold\n        print(f\"Fold {fold} | tau={tau:.2f} | score={score:.4f} | F1-bin={f1_bin:.4f} | F1-macro={f1_mac:.4f}\")\n\n    # 8) Tính OOF score với tau* = median(taus)\n    tau_star = float(np.median(taus))\n    y_oof_pred = _predict_with_tau(oof_proba, non_target_id, target_ids, tau_star)\n    overall_score, overall_binary_f1, overall_macro_f1 = competition_metric(y_train, y_oof_pred)\n\n    print(f\"\\n{'='*60}\")\n    print(\"CROSS-VALIDATION RESULTS (with tau*)\")\n    print(f\"{'='*60}\")\n    print(f\"Overall Competition Score: {overall_score:.4f} ± {np.std(cv_scores):.4f}\")\n    print(f\"Overall Binary F1: {overall_binary_f1:.4f}\")\n    print(f\"Overall Macro F1: {overall_macro_f1:.4f}\")\n    print(f\"Fold scores (per-fold best tau): {['{:.4f}'.format(s) for s in cv_scores]}\")\n    print(f\"tau* (median of folds): {tau_star:.2f}\")\n    print(f\"{'='*60}\")\n    print(\"✓ Training completed successfully!\")\n    print(\"✓ Models ready for inference (use prob-averaging + tau*)\")\n\n    return models, cv_scores, overall_score, tau_star, feature_cols\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.266834Z","iopub.execute_input":"2025-08-21T14:48:24.267177Z","iopub.status.idle":"2025-08-21T14:48:24.284828Z","shell.execute_reply.started":"2025-08-21T14:48:24.267150Z","shell.execute_reply":"2025-08-21T14:48:24.284265Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Dự đoán và Inference:","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# PREDICTION FUNCTION (with tau threshold & prob averaging) — PATCHED\n# =============================================================================\n\ndef create_prediction_function(models, feature_cols, imu_cols, tau_star, label2id, id2label, non_target_label='non_target'):\n    import numpy as np\n    import polars as pl\n\n    assert isinstance(models, (list, tuple)) and len(models) > 0, \"models must be a non-empty list\"\n    assert isinstance(feature_cols, (list, tuple)) and len(feature_cols) > 0, \"feature_cols must be provided\"\n    assert isinstance(imu_cols, (list, tuple)) and len(imu_cols) > 0, \"imu_cols must be provided\"\n    assert non_target_label in label2id, \"non_target label missing in label2id\"\n\n    non_target_id = int(label2id[non_target_label])\n    target_ids = {int(k) for k in id2label.keys() if int(k) != non_target_id}\n\n    # 18 NHÃN GỐC CỦA KAGGLE (chuẩn hóa format)\n    ALLOWED_LABELS = [\n        \"Above ear - pull hair\", \"Cheek - pinch skin\", \"Eyebrow - pull hair\", \"Eyelash - pull hair\",\n        \"Forehead - pull hairline\", \"Forehead - scratch\", \"Neck - pinch skin\", \"Neck - scratch\",\n        \"Drink from bottle/cup\", \"Put on earphones\", \"Read book\", \"Scratch elbow\", \"Scratch palm\",\n        \"Scratch with object\", \"Text on phone\", \"Touch back of head\", \"Write name in air\", \"Write name on leg\"\n    ]\n    NON_TARGET_FALLBACK = \"Drink from bottle/cup\"  # chọn 1 nhãn non‑target hợp lệ làm mặc định\n\n    def _predict_with_tau(proba_row, tau):\n        best_target_id = max(target_ids, key=lambda k: proba_row[k]) if target_ids else None\n        best_target_p = proba_row[best_target_id] if best_target_id is not None else 0.0\n        if best_target_p < tau:\n            return non_target_id\n        return int(np.argmax(proba_row))\n\n    def predict(sequence: pl.DataFrame, demographics: pl.DataFrame):\n        # 1) filter IMU cols\n        avail = sequence.columns\n        use_cols = [c for c in imu_cols if c in avail]\n        seq_pl = sequence.select(pl.col(use_cols))\n\n        # 2) featurize\n        feats = extract_comprehensive_features(seq_pl, demographics)   # -> pandas DF (1 row)\n\n        # 3) align columns\n        missing = [c for c in feature_cols if c not in feats.columns]\n        for c in missing:\n            feats[c] = 0.0\n        X = feats[feature_cols]\n\n        # 4) average probabilities\n        proba_sum = None\n        for m in models:\n            p = m.predict_proba(X)  # (1, n_classes)\n            proba_sum = p if proba_sum is None else (proba_sum + p)\n        proba_avg = proba_sum / len(models)\n\n        # 5) tau rule\n        y_id = _predict_with_tau(proba_avg[0], tau_star)\n        gesture = id2label[int(y_id)]\n\n        # ==== MAP VỀ 18 NHÃN HỢP LỆ (SỬA LỖI FORMAT) ====\n        # Nếu mô hình là 9 lớp (8 target + 'non_target'), khi ra 'non_target' => map về 1 nhãn hợp lệ.\n        if gesture == non_target_label:\n            gesture = NON_TARGET_FALLBACK\n        # Bảo hiểm: nếu vì lý do gì nhãn không đúng 18 nhãn gốc, ép về fallback\n        if gesture not in ALLOWED_LABELS:\n            gesture = NON_TARGET_FALLBACK\n        # =================================================\n\n        return gesture\n\n    return predict\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.285490Z","iopub.execute_input":"2025-08-21T14:48:24.285699Z","iopub.status.idle":"2025-08-21T14:48:24.302133Z","shell.execute_reply.started":"2025-08-21T14:48:24.285685Z","shell.execute_reply":"2025-08-21T14:48:24.301441Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Hàm Tính toán World Acceleration","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# CORE FEATURE ENGINEERING: WORLD ACCELERATION\n# =============================================================================\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    \"\"\"\n    Convert acceleration from device coordinates to world coordinates\n    \"\"\"\n    try:\n        # Convert quaternion format from [w, x, y, z] to [x, y, z, w] for scipy\n        rot_scipy = rot[:, [1, 2, 3, 0]]\n        \n        # Verify quaternions are valid (non-zero norm)\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        if np.any(norms < 1e-8):\n            rot_scipy[norms < 1e-8] = [0.0, 0.0, 0.0, 1.0]  # Identity quaternion in scipy format\n        \n        # Create rotation object and apply transformation\n        r = R.from_quat(rot_scipy)\n        acc_world = r.apply(acc)\n        \n    except Exception:\n        # Fallback to original acceleration if transformation fails\n        acc_world = acc.copy()\n    \n    return acc_world","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.302920Z","iopub.execute_input":"2025-08-21T14:48:24.303157Z","iopub.status.idle":"2025-08-21T14:48:24.316502Z","shell.execute_reply.started":"2025-08-21T14:48:24.303135Z","shell.execute_reply":"2025-08-21T14:48:24.315965Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Features","metadata":{}},{"cell_type":"code","source":"def extract_statistical_features(data: np.ndarray, prefix: str) -> dict:\n    \"\"\"\n    Extract comprehensive statistical features from a 1D time series.\n    This function computes common statistical features like mean, std, skewness, kurtosis, etc.\n    \n    Returns features that capture:\n    - Central tendency: mean, median, mode region\n    - Spread: std, variance, range, IQR\n    - Shape: skewness, kurtosis\n    - Dynamics: differences, trends, changes\n    - Segments: beginning vs middle vs end behavior\n    \"\"\"\n    \n    features = {}\n    \n    # Basic statistics\n    features[f'{prefix}_mean'] = np.mean(data)\n    features[f'{prefix}_std'] = np.std(data)\n    features[f'{prefix}_var'] = np.var(data)\n    features[f'{prefix}_min'] = np.min(data)\n    features[f'{prefix}_max'] = np.max(data)\n    features[f'{prefix}_median'] = np.median(data)\n    features[f'{prefix}_q25'] = np.percentile(data, 25)\n    features[f'{prefix}_q75'] = np.percentile(data, 75)\n    features[f'{prefix}_iqr'] = np.percentile(data, 75) - np.percentile(data, 25)\n    \n    # Range and boundary features\n    features[f'{prefix}_range'] = np.max(data) - np.min(data)\n    features[f'{prefix}_first'] = data[0] if len(data) > 0 else 0\n    features[f'{prefix}_last'] = data[-1] if len(data) > 0 else 0\n    features[f'{prefix}_delta'] = data[-1] - data[0] if len(data) > 0 else 0\n    \n    # Higher order moments (shape of distribution)\n    if len(data) > 1 and np.std(data) > 1e-8:\n        features[f'{prefix}_skew'] = pd.Series(data).skew()\n        features[f'{prefix}_kurt'] = pd.Series(data).kurtosis()\n    else:\n        features[f'{prefix}_skew'] = 0\n        features[f'{prefix}_kurt'] = 0\n    \n    # Differential features (capture dynamics)\n    if len(data) > 1:\n        diff_data = np.diff(data)\n        features[f'{prefix}_diff_mean'] = np.mean(diff_data)\n        features[f'{prefix}_diff_std'] = np.std(diff_data)\n        features[f'{prefix}_n_changes'] = np.sum(np.abs(diff_data) > np.std(data) * 0.1)  # Significant changes\n    else:\n        features[f'{prefix}_diff_mean'] = 0\n        features[f'{prefix}_diff_std'] = 0\n        features[f'{prefix}_n_changes'] = 0\n    \n    # Correlation with time (trend detection)\n    if len(data) > 2:\n        time_indices = np.arange(len(data))\n        try:\n            corr_coef = np.corrcoef(time_indices, data)[0, 1]\n            features[f'{prefix}_time_corr'] = corr_coef if not np.isnan(corr_coef) else 0\n        except:\n            features[f'{prefix}_time_corr'] = 0\n    else:\n        features[f'{prefix}_time_corr'] = 0\n    \n    # Segment features (beginning, middle, end patterns)\n    seq_len = len(data)\n    if seq_len >= 9:  # Need sufficient data for meaningful segments\n        seg_size = seq_len // 3\n        seg1 = data[:seg_size]           # Beginning (Transition phase)\n        seg2 = data[seg_size:2*seg_size] # Middle (Pause phase)  \n        seg3 = data[2*seg_size:]         # End (Gesture phase)\n        \n        features[f'{prefix}_seg1_mean'] = np.mean(seg1)\n        features[f'{prefix}_seg2_mean'] = np.mean(seg2)\n        features[f'{prefix}_seg3_mean'] = np.mean(seg3)\n        \n        features[f'{prefix}_seg1_std'] = np.std(seg1)\n        features[f'{prefix}_seg2_std'] = np.std(seg2)\n        features[f'{prefix}_seg3_std'] = np.std(seg3)\n        \n        # Segment transitions (important for distinguishing gesture types)\n        features[f'{prefix}_seg1_to_seg2'] = np.mean(seg2) - np.mean(seg1)\n        features[f'{prefix}_seg2_to_seg3'] = np.mean(seg3) - np.mean(seg2)\n    else:\n        # Not enough data for meaningful segments\n        for seg in [1, 2, 3]:\n            features[f'{prefix}_seg{seg}_mean'] = features[f'{prefix}_mean']\n            features[f'{prefix}_seg{seg}_std'] = features[f'{prefix}_std']\n        features[f'{prefix}_seg1_to_seg2'] = 0\n        features[f'{prefix}_seg2_to_seg3'] = 0\n    \n    return features\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.317175Z","iopub.execute_input":"2025-08-21T14:48:24.317362Z","iopub.status.idle":"2025-08-21T14:48:24.339010Z","shell.execute_reply.started":"2025-08-21T14:48:24.317348Z","shell.execute_reply":"2025-08-21T14:48:24.338457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef competition_metric(y_true, y_pred) -> tuple:\n    \"\"\"Calculate the competition metric (Binary F1 + Macro F1) / 2\"\"\"\n    \n    # Binary F1: BFRB vs non-BFRB (classifying BFRB gestures as 1, others as 0)\n    binary_f1 = f1_score(\n        np.where(y_true <= 7, 1, 0),\n        np.where(y_pred <= 7, 1, 0),\n        zero_division=0.0,\n    )\n    \n    # Macro F1: Calculating F1 score across all BFRB gestures (0-7)\n    macro_f1 = f1_score(\n        np.where(y_true <= 7, y_true, 99),  # Map non-BFRB gestures to class 99\n        np.where(y_pred <= 7, y_pred, 99),  # Map non-BFRB gestures to class 99\n        average=\"macro\", \n        zero_division=0.0,\n    )\n    \n    # Final competition score: average of binary F1 and macro F1\n    final_score = 0.5 * (binary_f1 + macro_f1)\n    \n    return final_score, binary_f1, macro_f1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.339691Z","iopub.execute_input":"2025-08-21T14:48:24.339847Z","iopub.status.idle":"2025-08-21T14:48:24.348369Z","shell.execute_reply.started":"2025-08-21T14:48:24.339835Z","shell.execute_reply":"2025-08-21T14:48:24.347732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    \"\"\"\n    Handle missing values in quaternion data intelligently\n    \n    Key insight: Quaternions must have unit length |q| = 1\n    If one component is missing, we can reconstruct it from the others\n    \"\"\"\n    rot_cleaned = rot_data.copy()\n    \n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        \n        if missing_count == 0:\n            # No missing values, normalize to unit quaternion\n            norm = np.linalg.norm(row)\n            if norm > 1e-8:\n                rot_cleaned[i] = row / norm\n            else:\n                rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]  # Identity quaternion\n                \n        elif missing_count == 1:\n            # One missing value, reconstruct using unit quaternion constraint\n            # |w|² + |x|² + |y|² + |z|² = 1\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            \n            sum_squares = np.sum(valid_values**2)\n            if sum_squares <= 1.0:\n                missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n                # Choose sign for continuity with previous quaternion\n                if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n                    if rot_cleaned[i-1, missing_idx] < 0:\n                        missing_value = -missing_value\n                rot_cleaned[i, missing_idx] = missing_value\n                rot_cleaned[i, ~np.isnan(row)] = valid_values\n            else:\n                rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n        else:\n            # More than one missing value, use identity quaternion\n            rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n    \n    return rot_cleaned\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.351751Z","iopub.execute_input":"2025-08-21T14:48:24.351936Z","iopub.status.idle":"2025-08-21T14:48:24.360379Z","shell.execute_reply.started":"2025-08-21T14:48:24.351922Z","shell.execute_reply":"2025-08-21T14:48:24.359656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# SIMPLIFIED FEATURE EXTRACTION\n# =============================================================================\n\ndef extract_comprehensive_features(sequence: pl.DataFrame, demographics: pl.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Extract features from IMU data with world acceleration transformation.\n    This function should return a DataFrame with extracted features.\n    \"\"\"\n    # Convert to pandas for processing\n    seq_df = sequence.to_pandas()\n    demo_df = demographics.to_pandas()\n    \n    # Handle missing values in basic sensor data\n    acc_data = seq_df[Config.ACC_COLS].copy()\n    acc_data = acc_data.ffill().bfill().fillna(0)\n    \n    rot_data = seq_df[Config.ROT_COLS].copy()\n    rot_data = rot_data.ffill().bfill()\n    \n    # Handle quaternion missing values and normalize\n    rot_data_clean = handle_quaternion_missing_values(rot_data.values)\n    \n    # CORE INNOVATION: Compute world acceleration\n    try:\n        world_acc_data = compute_world_acceleration(acc_data.values, rot_data_clean)\n    except Exception as e:\n        print(f\"Warning: World acceleration computation failed: {e}\")\n        world_acc_data = acc_data.values.copy()  # Fallback to device coordinates\n    \n    # Initialize feature dictionary\n    features = {}\n    \n    # Add sequence metadata\n    features['sequence_length'] = len(seq_df)\n    \n    # Add demographics features\n    if len(demo_df) > 0:\n        demo_row = demo_df.iloc[0]\n        features['age'] = demo_row.get('age', 0)\n        features['adult_child'] = demo_row.get('adult_child', 0)\n        features['sex'] = demo_row.get('sex', 0)\n        features['handedness'] = demo_row.get('handedness', 0)\n        features['height_cm'] = demo_row.get('height_cm', 0)\n        features['shoulder_to_wrist_cm'] = demo_row.get('shoulder_to_wrist_cm', 0)\n        features['elbow_to_wrist_cm'] = demo_row.get('elbow_to_wrist_cm', 0)\n    \n    # Define feature arrays for statistical extraction\n    feature_arrays = {\n        'acc': acc_data.values,           # Device acceleration (3D)\n        'rot': rot_data_clean,            # Rotation quaternion (4D) \n        'world_acc': world_acc_data,      # World acceleration (3D) - KEY INNOVATION\n    }\n    \n    # Extract statistical features for each data source\n    for source_name, array in feature_arrays.items():\n        if array.ndim == 1:\n            array = array.reshape(-1, 1)\n        \n        n_features = array.shape[1]\n        \n        for feat_idx in range(n_features):\n            feat_data = array[:, feat_idx]\n            \n            # Create feature name\n            if source_name == 'acc':\n                axis_names = ['x', 'y', 'z']\n                prefix = f\"acc_{axis_names[feat_idx]}\"\n            elif source_name == 'rot':\n                comp_names = ['w', 'x', 'y', 'z']\n                prefix = f\"rot_{comp_names[feat_idx]}\"\n            elif source_name == 'world_acc':\n                axis_names = ['x', 'y', 'z']  \n                prefix = f\"world_acc_{axis_names[feat_idx]}\"\n            else:\n                prefix = f\"{source_name}_{feat_idx}\" if n_features > 1 else source_name\n            \n            # Extract comprehensive statistical features\n            features.update(extract_statistical_features(feat_data, prefix))\n    \n    # Compute magnitude features (important for motion intensity)\n    acc_magnitude = np.linalg.norm(acc_data.values, axis=1)\n    world_acc_magnitude = np.linalg.norm(world_acc_data, axis=1)\n    \n    features.update(extract_statistical_features(acc_magnitude, 'acc_magnitude'))\n    features.update(extract_statistical_features(world_acc_magnitude, 'world_acc_magnitude'))\n    \n    # Cross-feature: difference between device and world acceleration magnitudes\n    acc_world_diff = acc_magnitude - world_acc_magnitude\n    features.update(extract_statistical_features(acc_world_diff, 'acc_world_diff'))\n    \n    # Convert to DataFrame\n    result_df = pd.DataFrame([features])\n    \n    # Handle any remaining NaN values\n    result_df = result_df.fillna(0)\n    \n    return result_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.360974Z","iopub.execute_input":"2025-08-21T14:48:24.361170Z","iopub.status.idle":"2025-08-21T14:48:24.371711Z","shell.execute_reply.started":"2025-08-21T14:48:24.361155Z","shell.execute_reply":"2025-08-21T14:48:24.371183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Chuẩn bị và tải dữ liệu:","metadata":{}},{"cell_type":"code","source":"def load_and_prepare_data():\n    \"\"\"Load and prepare training/test features (IMU-only) + grouped labels\"\"\"\n    print(\"Loading training data...\")\n    train_df = pl.read_csv(Config.TRAIN_PATH)\n    train_demographics = pl.read_csv(Config.TRAIN_DEMOGRAPHICS_PATH)\n\n    print(\"Loading test data...\")\n    test_df = pl.read_csv(Config.TEST_PATH)\n    test_demographics = pl.read_csv(Config.TEST_DEMOGRAPHICS_PATH)\n\n    # Common IMU columns (exclude ToF/Thermal)\n    train_cols = set(train_df.columns)\n    test_cols = set(test_df.columns)\n    common_cols = train_cols.intersection(test_cols)\n    imu_cols = [c for c in common_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n\n    print(f\"✓ Using {len(imu_cols)} common IMU columns\")\n    print(f\"✓ Train-only columns: {train_cols - test_cols}\")\n    print(f\"✓ Test-only columns: {test_cols - train_cols}\")\n\n    # =========================\n    # TRAIN FEATURE EXTRACTION\n    # =========================\n    print(\"Extracting features for training sequences...\")\n    train_features_list = []\n    y_train_grouped = []\n    train_subjects = []\n    train_sequence_ids = []\n\n    # Cần có subject + sequence_id + gesture để gán nhãn/nhóm\n    base_train_cols = list(set(imu_cols) | {'subject','sequence_id','gesture'})\n    train_sequences = train_df.select(pl.col(base_train_cols)).group_by('sequence_id', maintain_order=True)\n\n    for seq_key, seq_pl in train_sequences:\n        # Lấy sequence_id & subject\n        sequence_id_val = seq_key[0] if isinstance(seq_key, tuple) else seq_key\n        subject_id = seq_pl['subject'][0]\n        subject_demo = train_demographics.filter(pl.col('subject') == subject_id)\n\n        # IMU-only cho feature extraction\n        imu_only_pl = seq_pl.select(pl.col(imu_cols))\n        features_pd = extract_comprehensive_features(imu_only_pl, subject_demo)  # -> pandas DataFrame\n        features_pd['sequence_id'] = sequence_id_val\n        train_features_list.append(features_pd)\n\n        # Nhãn gốc -> nhãn gộp\n        gesture_name = seq_pl['gesture'][0]\n        grouped_label = gesture_name if gesture_name in TARGET_LABELS else 'non_target'\n        y_train_grouped.append(label2id[grouped_label])\n\n        train_subjects.append(subject_id)\n        train_sequence_ids.append(sequence_id_val)\n\n    X_train = pd.concat(train_features_list, ignore_index=True)\n    y_train = np.array(y_train_grouped, dtype=int)\n    subjects = np.array(train_subjects)\n\n    # =========================\n    # TEST FEATURE EXTRACTION\n    # =========================\n    print(\"Extracting features for test sequences...\")\n    test_features_list = []\n    test_sequence_ids = []\n\n    base_test_cols = list(set(imu_cols) | {'subject','sequence_id'})\n    test_sequences = test_df.select(pl.col(base_test_cols)).group_by('sequence_id', maintain_order=True)\n\n    for seq_key, seq_pl in test_sequences:\n        sequence_id_val = seq_key[0] if isinstance(seq_key, tuple) else seq_key\n        subject_id = seq_pl['subject'][0]\n        subject_demo = test_demographics.filter(pl.col('subject') == subject_id)\n\n        # Lưu ý: ở test không có gesture\n        imu_only_pl = seq_pl.select(pl.col(imu_cols))\n        features_pd = extract_comprehensive_features(imu_only_pl, subject_demo)\n        features_pd['sequence_id'] = sequence_id_val\n\n        test_features_list.append(features_pd)\n        test_sequence_ids.append(sequence_id_val)\n\n    X_test = pd.concat(test_features_list, ignore_index=True)\n\n    print(f\"✓ Training features shape: {X_train.shape}\")\n    print(f\"✓ Training labels shape: {y_train.shape}\")\n    print(f\"✓ Test features shape: {X_test.shape}\")\n    print(f\"✓ Number of features: {X_train.shape[1] - 1}\")  # -1 for sequence_id\n\n    # Bỏ sequence_id khỏi ma trận đặc trưng (giữ riêng để debug nếu muốn)\n    X_train = X_train.drop(columns=['sequence_id'], errors='ignore')\n    X_test  = X_test.drop(columns=['sequence_id'], errors='ignore')\n\n    return X_train, y_train, subjects, X_test, test_sequence_ids, imu_cols\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.372346Z","iopub.execute_input":"2025-08-21T14:48:24.372619Z","iopub.status.idle":"2025-08-21T14:48:24.390222Z","shell.execute_reply.started":"2025-08-21T14:48:24.372597Z","shell.execute_reply":"2025-08-21T14:48:24.389509Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference Server Setup:","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# INFERENCE SERVER SETUP\n# =============================================================================\n\ndef setup_inference_server(predict_function):\n    \"\"\"Set up the inference server to serve predictions\"\"\"\n    inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict_function)\n    \n    # If in competition rerun, use Kaggle environment to serve\n    if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n        inference_server.serve()\n    else:\n        inference_server.run_local_gateway(data_paths=(Config.TEST_PATH, Config.TEST_DEMOGRAPHICS_PATH))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.390914Z","iopub.execute_input":"2025-08-21T14:48:24.391224Z","iopub.status.idle":"2025-08-21T14:48:24.403453Z","shell.execute_reply.started":"2025-08-21T14:48:24.391208Z","shell.execute_reply":"2025-08-21T14:48:24.402904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Chạy Toàn Bộ Pipeline:","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# MAIN EXECUTION PIPELINE (compatible with tau* + grouped labels)\n# =============================================================================\n\ndef main():\n    X_train, y_train, subjects, X_test, test_sequence_ids, imu_cols = load_and_prepare_data()\n\n    models, cv_scores, overall_score, tau_star, feature_cols = train_lightgbm_models(\n        X_train=X_train,\n        y_train=y_train,\n        subjects=subjects\n    )\n\n    # Tạo predict_function (đủ tham số)\n    predict_function = create_prediction_function(\n        models=models,\n        feature_cols=feature_cols,\n        imu_cols=imu_cols,\n        tau_star=tau_star,\n        label2id=label2id,    # mapping GỘP (có 'non_target')\n        id2label=id2label,    # mapping GỘP\n        non_target_label='non_target'\n    )\n    print(\"predict_function is callable?\", callable(predict_function))  # phải là True\n\n    # Khởi tạo server (chỉ khi callable)\n    server = setup_inference_server(predict_function)\n\n    print(\"\\n✓ Training completed successfully!\")\n    print(f\"✓ Final CV Score (OOF, tau*): {overall_score:.4f}\")\n    return predict_function, models, cv_scores\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.404152Z","iopub.execute_input":"2025-08-21T14:48:24.404385Z","iopub.status.idle":"2025-08-21T14:48:24.416290Z","shell.execute_reply.started":"2025-08-21T14:48:24.404362Z","shell.execute_reply":"2025-08-21T14:48:24.415596Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Chạy Pipeline Chính","metadata":{}},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    predict_function, trained_models, cv_scores = main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T14:48:24.416966Z","iopub.execute_input":"2025-08-21T14:48:24.417197Z","iopub.status.idle":"2025-08-21T14:55:40.635734Z","shell.execute_reply.started":"2025-08-21T14:48:24.417174Z","shell.execute_reply":"2025-08-21T14:55:40.635006Z"}},"outputs":[],"execution_count":null}]}