{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Tổng quan","metadata":{}},{"cell_type":"markdown","source":"Mục tiêu là khám phá các thuật toán học máy giám sát để phát hiện hành vi dựa trên dữ liệu cảm biến. Chúng ta sẽ xử lý các vấn đề như giá trị thiếu, xây dựng đường ống (pipelines), tạo ra sơ đồ phụ thuộc, và áp dụng mã hóa một lần.\n\nChúng ta sẽ sử dụng dữ liệu từ các cảm biến để dự đoán hành vi của các đối tượng trong môi trường. Các thuật toán học máy sẽ giúp phân loại hành vi dựa trên các đặc điểm đo lường được từ dữ liệu cảm biến. Mục tiêu cuối cùng là xây dựng một mô hình có khả năng dự đoán chính xác hành vi dựa trên các tính năng cảm biến được cung cấp.","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Tải dữ liệu và thư viện","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings\nfrom matplotlib.ticker import MaxNLocator\nfrom mpl_toolkits.mplot3d import Axes3D\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nfrom sklearn.metrics import accuracy_score, f1_score\nimport joblib\nfrom scipy.spatial.transform import Rotation as R","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:48:40.160594Z","iopub.execute_input":"2025-08-11T13:48:40.161775Z","iopub.status.idle":"2025-08-11T13:48:40.170903Z","shell.execute_reply.started":"2025-08-11T13:48:40.161722Z","shell.execute_reply":"2025-08-11T13:48:40.169652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tải tập dữ liệu\n#Dữ liệu cảm biến cho tập huấn luyện (Train Sensor Data): \ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\n#Thông tin nhân khẩu học của đối tượng trong tập huấn luyện (giới tính, tuổi, ...): \ntrain_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\n\n#Dữ liệu cảm biến cho tập kiểm tra:\ntest_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\n\n# Thông tin nhân khẩu học của đối tượng trong tập kiểm tra:\ntest_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:48:40.173173Z","iopub.execute_input":"2025-08-11T13:48:40.173471Z","iopub.status.idle":"2025-08-11T13:49:05.683950Z","shell.execute_reply.started":"2025-08-11T13:48:40.173446Z","shell.execute_reply":"2025-08-11T13:49:05.682831Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.1 Khám phá dữ liệu","metadata":{}},{"cell_type":"code","source":"datasets = {\n    \"Train Data\": train_df,\n    \"Train Demographics\": train_dem_df,\n    \"Test Data\": test_df,\n    \"Test Demographics\": test_dem_df,\n}\n\nfor name, df in datasets.items():\n    print(f\"Missing values in {name}:\")\n    missing = df.isnull().sum()\n    missing = missing[missing > 0]\n    if not missing.empty:\n        print(missing)\n    else:\n        print(\"  ✅ No missing values.\")\n    print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:05.684984Z","iopub.execute_input":"2025-08-11T13:49:05.685372Z","iopub.status.idle":"2025-08-11T13:49:06.649298Z","shell.execute_reply.started":"2025-08-11T13:49:05.685341Z","shell.execute_reply":"2025-08-11T13:49:06.648289Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.5 kiểm tra giá trị trùng lặp","metadata":{}},{"cell_type":"code","source":"#Kiểm tra trùng lặp (Duplicate Rows): .duplicated(): Trả về Series boolean, dòng nào trùng lặp sẽ là True.\n#                                     .sum(): Đếm tổng số dòng trùng lặp.\n#Kiểm tra xem dữ liệu có bản ghi nào bị trùng không — quan trọng cho việc làm sạch dữ liệu.\n\n# Đếm các hàng trùng lặp trong train_df\ntrain_duplicates = train_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong  test_df\ntest_duplicates = test_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong train_dem_df (optional)\ntrain_dem_duplicates = train_dem_df.duplicated().sum()\n# Đếm các hàng trùng lặp trong test_dem_df (optional)\ntest_dem_duplicates = test_dem_df.duplicated().sum()\n\n# In số lượng dòng trùng:\nprint(f\"Number of duplicate rows in train_df: {train_duplicates}\")\nprint(f\"Number of duplicate rows in test_df: {test_duplicates}\")\nprint(f\"Number of duplicate rows in train_dem_df: {train_dem_duplicates}\")\nprint(f\"Number of duplicate rows in test_dem_df: {test_dem_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:06.650318Z","iopub.execute_input":"2025-08-11T13:49:06.650733Z","iopub.status.idle":"2025-08-11T13:49:13.685695Z","shell.execute_reply.started":"2025-08-11T13:49:06.650594Z","shell.execute_reply":"2025-08-11T13:49:13.682474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"markdown","source":"#### UTILS/HELPERS","metadata":{}},{"cell_type":"code","source":"def null_percent(df):\n    per=((df.isnull().sum()/len(df))*100).round(2)\n    return per[per>0]\n\nprint(\"Nan Values in Train data\")\nprint(null_percent(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:13.696244Z","iopub.execute_input":"2025-08-11T13:49:13.697107Z","iopub.status.idle":"2025-08-11T13:49:15.520782Z","shell.execute_reply.started":"2025-08-11T13:49:13.697083Z","shell.execute_reply":"2025-08-11T13:49:15.519809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Các cột có tỷ lệ NaN cao nhất là các cột có tên bắt đầu với tof_5_v (5.24% NaN).\n\nCác cột khác như rot_x, rot_w có tỷ lệ NaN thấp hơn nhiều (0.64%).","metadata":{}},{"cell_type":"markdown","source":"## 2.2 Quan sát hoặc xu hướng ban đầu","metadata":{}},{"cell_type":"code","source":"# --- 2.4 Hợp nhất dữ liệu nhân khẩu học và cảm biến ---\nmerged_train = pd.merge(train_df, train_dem_df, on=\"subject\", how=\"left\")\nmerged_test = pd.merge(test_df, test_dem_df, on=\"subject\", how=\"left\")\n\nprint(\"Merged Train Shape:\", merged_train.shape)\nprint(\"Merged Test Shape:\", merged_test.shape)\ndisplay(merged_train.head(2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:15.521865Z","iopub.execute_input":"2025-08-11T13:49:15.522244Z","iopub.status.idle":"2025-08-11T13:49:16.325427Z","shell.execute_reply.started":"2025-08-11T13:49:15.522219Z","shell.execute_reply":"2025-08-11T13:49:16.324498Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Merged Train Shape:** (574945, 348) - Sau khi hợp nhất, merged_train có 574,945 dòng và 348 cột, tức là dữ liệu từ train_df (574,945 dòng và 341 cột) đã được kết hợp với 7 cột từ train_dem_df (vì train_dem_df có 8 cột, nhưng 1 cột là subject đã được sử dụng làm khóa hợp nhất).\n\n**Merged Test Shape:** (107, 343) - Tương tự, merged_test có 107 dòng và 343 cột, kết quả là hợp nhất giữa test_df (107 dòng và 336 cột) với test_dem_df (8 cột, với 1 cột subject là khóa hợp nhất).","metadata":{}},{"cell_type":"code","source":"# --- Thống kê mô tả toàn bộ merged_train ---\nmerged_train.describe().T\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:16.326407Z","iopub.execute_input":"2025-08-11T13:49:16.326668Z","iopub.status.idle":"2025-08-11T13:49:27.800000Z","shell.execute_reply.started":"2025-08-11T13:49:16.326646Z","shell.execute_reply":"2025-08-11T13:49:27.798811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra thiếu giá trị & thống kê cảm biến: \n#Danh sách các cột không thuộc cảm biến (có thể là ID, thông tin khác). \n\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nsensor_cols = [col for col in train_df.columns if not col.startswith(excluded_prefixes)]\n\n# Sensor Data Summary for TRAIN\n#isnull().sum(): Đếm số giá trị bị thiếu.\nmissing_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    '[TRAIN] Missing Count': train_df[sensor_cols].isnull().sum().values,\n    '[TRAIN] Missing %': (train_df[sensor_cols].isnull().sum().values / len(train_df)) * 100\n})\n\n#nunique(): Đếm số lượng giá trị duy nhất.\nunique_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Unique Values [TRAIN]': train_df[sensor_cols].nunique().values\n})\n\n#dtypes: Lấy kiểu dữ liệu của từng cột.\ndtypes_sensor = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Data Type': train_df[sensor_cols].dtypes.values\n})\n\n# Merge all summaries (NO test set)\n#merge: Gộp các bảng thống kê thành bảng duy nhất theo Feature.\nsensor_summary = missing_sensor_train \\\n    .merge(unique_sensor_train, on='Feature', how='left') \\\n    .merge(dtypes_sensor, on='Feature', how='left')\n\n# Display styled DataFrame (mask NaNs just for styling)\n#fillna(0): Điền giá trị thiếu bằng 0 (cho đẹp mắt khi hiển thị).\n#.style.background_gradient: Tô màu nền theo giá trị giúp dễ nhìn.\nstyled_df = sensor_summary.fillna(0)\nstyled_df.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:27.801100Z","iopub.execute_input":"2025-08-11T13:49:27.801410Z","iopub.status.idle":"2025-08-11T13:49:28.883830Z","shell.execute_reply.started":"2025-08-11T13:49:27.801376Z","shell.execute_reply":"2025-08-11T13:49:28.882960Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Không có giá trị thiếu: Tất cả các cột trong bảng thống kê đều không có giá trị thiếu (Missing Count = 0), cho thấy dữ liệu của bạn rất đầy đủ.\n\n- Sự đa dạng trong dữ liệu: Các cột như row_id, sequence_id, và subject có số lượng giá trị duy nhất khá lớn, trong khi các cột như sequence_type, phase, và behavior có ít giá trị duy nhất hơn, cho thấy chúng có thể là các thông tin phân loại.\n\n- Kiểu dữ liệu: Hầu hết các cột đều có kiểu dữ liệu object, ngoại trừ sequence_counter có kiểu int64, điều này cho thấy hầu hết các cột chứa thông tin phân loại (string), chỉ có một cột chứa giá trị số nguyên.","metadata":{}},{"cell_type":"code","source":"#Thống kê tương tự cho nhân khẩu học: Tương tự như phần thống kê cảm biến nhưng áp dụng cho dữ liệu nhân khẩu học.\n\n# Cột nhân khẩu học (không loại trừ)\ndem_cols = train_dem_df.columns\n\n# Giá trị bị thiếu trong dữ liệu nhân khẩu học của train \nmissing_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    '[TRAIN DEMO] Missing Count': train_dem_df[dem_cols].isnull().sum().values,\n    '[TRAIN DEMO] Missing %': (train_dem_df[dem_cols].isnull().sum().values / len(train_dem_df)) * 100\n})\n\n# Giá trị duy nhất được tính trong dữ liệu nhân khẩu học của train \nunique_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    'Unique Values [TRAIN DEMO]': train_dem_df[dem_cols].nunique().values\n})\n\n# Data types\ndtypes_demo = pd.DataFrame({\n    'Feature': dem_cols,\n    'Data Type': train_dem_df[dem_cols].dtypes.values\n})\n\n# Tóm tắt hợp nhất (chỉ dành cho đào tạo)\ndemo_summary = (\n    missing_demo_train\n    .merge(unique_demo_train, on='Feature', how='left')\n    .merge(dtypes_demo, on='Feature', how='left')\n)\n\n# Hiển thị tóm tắt theo phong cách\ndemo_summary.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:28.884817Z","iopub.execute_input":"2025-08-11T13:49:28.885152Z","iopub.status.idle":"2025-08-11T13:49:28.911483Z","shell.execute_reply.started":"2025-08-11T13:49:28.885114Z","shell.execute_reply":"2025-08-11T13:49:28.910589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Không có giá trị thiếu: Các cột nhân khẩu học đều không có giá trị thiếu, cho thấy dữ liệu đầy đủ và sẵn sàng để phân tích.\n\n- Sự đa dạng trong dữ liệu: Các cột như subject, age, và height_cm có sự đa dạng về giá trị duy nhất, trong khi các cột như sex, handedness có ít giá trị duy nhất hơn, cho thấy đây là các cột phân loại.\n\n- Kiểu dữ liệu: Các cột phần lớn có kiểu dữ liệu int64 hoặc float64, cho thấy dữ liệu chủ yếu là số nguyên hoặc số thực. Các cột như subject có kiểu dữ liệu object, có thể là các chuỗi ký tự.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# 1) Sao chép đào tạo và kiểm tra để chúng ta không sửa đổi DataFrame gốc\ntrain_temp = train_df.copy()\ntest_temp  = test_df.copy()\n\n# 2) GIA TỐC KẾ: tính toán độ lớn tại mỗi dấu thời gian\ntrain_temp['acc_mag'] = np.sqrt(\n    train_temp['acc_x']**2 + train_temp['acc_y']**2 + train_temp['acc_z']**2\n)\ntest_temp['acc_mag'] = np.sqrt(\n    test_temp['acc_x']**2 + test_temp['acc_y']**2 + test_temp['acc_z']**2\n)\n\n# 3) ROTATION: tính toán “góc quay” từ thành phần quaternion w\n# (Lưu ý: rot_w nằm trong [-1,1], do đó arccos hợp lệ. Chúng tôi bỏ qua NaN nếu có.)\ntrain_temp['rot_angle'] = 2 * np.arccos(train_temp['rot_w'].clip(-1,1))\ntest_temp['rot_angle']  = 2 * np.arccos(test_temp['rot_w'].clip(-1,1))\n\n# 4) Nhóm theo sequence_id và tổng hợp các tóm tắt gia tốc kế\nacc_agg_funcs = {\n    'acc_mag': ['mean', 'std', 'max']\n}\ntrain_acc_summary = train_temp.groupby('sequence_id').agg(acc_agg_funcs)\ntest_acc_summary  = test_temp.groupby('sequence_id').agg(acc_agg_funcs)\n\n# Làm phẳng cột MultiIndex\ntrain_acc_summary.columns = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\ntest_acc_summary.columns  = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\n\n# 5) Nhóm theo sequence_id và tổng hợp tóm tắt vòng quay\nrot_agg_funcs = {\n    'rot_angle': ['mean', 'std', 'max']\n}\ntrain_rot_summary = train_temp.groupby('sequence_id').agg(rot_agg_funcs)\ntest_rot_summary  = test_temp.groupby('sequence_id').agg(rot_agg_funcs)\n\ntrain_rot_summary.columns = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\ntest_rot_summary.columns  = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\n\n# 6) NHIỆT ĐỘ: năm cảm biến thm_1 … thm_5\nthm_cols = [f\"thm_{i}\" for i in range(1, 6)]\n\n# Xác định hàm tổng hợp: trung bình + độ lệch chuẩn\nthm_agg_funcs = {col: ['mean', 'std'] for col in thm_cols}\n\ntrain_thm_summary = train_temp.groupby('sequence_id').agg(thm_agg_funcs)\ntest_thm_summary  = test_temp.groupby('sequence_id').agg(thm_agg_funcs)\n\n# Làm phẳng các cột MultiIndex\nflattened_thm_cols = []\nfor sensor in thm_cols:\n    for stat in ['mean','std']:\n        flattened_thm_cols.append(f\"{sensor}_{stat}\")\n\ntrain_thm_summary.columns = flattened_thm_cols\ntest_thm_summary.columns  = flattened_thm_cols\n\n# 7) THỜI GIAN BAY: mỗi cảm biến i có 64 cột pixel: tof_i_v0 … tof_i_v63\n# Chúng ta sẽ tạo một “tof_i_mean_at_ts” cho mỗi dấu thời gian, sau đó tổng hợp theo từng chuỗi.\n\ndef compute_tof_sequence_summary(df):\n    # Khởi tạo một dict để giữ DataFrames theo từng chuỗi\n    seq_summaries = {}\n\n    for i in range(1, 6):\n        # Xây dựng danh sách các cột cho cảm biến i\n        tof_cols = [f\"tof_{i}_v{pix}\" for pix in range(64)]\n        # Thay thế -1 bằng NaN để chúng không làm lệch giá trị trung bình; chuyển đổi sang float\n        ts_grid = df[tof_cols].replace(-1, np.nan).astype(float)\n        # Tính toán “trung bình trên tất cả 64 pixel” cho mỗi dấu thời gian\n        df[f\"tof_{i}_mean_at_ts\"] = ts_grid.mean(axis=1)\n    \n   # Bây giờ, nhóm theo id chuỗi và tính giá trị trung bình và độ lệch chuẩn của các giá trị trung bình đó\n    agg_dict = {f\"tof_{i}_mean_at_ts\": ['mean','std'] for i in range(1, 6)}\n    summary = df.groupby('sequence_id').agg(agg_dict)\n    # Làm phẳng các cột MultiIndex\n    flat_cols = [f\"tof_{i}_{stat}\" for i in range(1, 6) for stat in ['mean','std']]\n    summary.columns = flat_cols\n    return summary\n\ntrain_tof_summary = compute_tof_sequence_summary(train_temp)\ntest_tof_summary  = compute_tof_sequence_summary(test_temp)\n\n# 8) Hợp nhất các tóm tắt accel, rotation, thm, tof (trên sequence_id)\ntrain_sensor_summary = (\n    train_acc_summary\n    .join(train_rot_summary, how='outer')\n    .join(train_thm_summary, how='outer')\n    .join(train_tof_summary, how='outer')\n)\n\ntest_sensor_summary = (\n    test_acc_summary\n    .join(test_rot_summary, how='outer')\n    .join(test_thm_summary, how='outer')\n    .join(test_tof_summary, how='outer')\n)\n\n# 9) Thêm cột “Dataset” để chúng ta có thể thực hiện box+hist song song\ntrain_sensor_summary['Dataset'] = 'Train'\ntest_sensor_summary['Dataset']  = 'Test'\n\n# 10) Ghép nối thành một DataFrame để vẽ đồ thị\ncombined_sensor_summary = pd.concat(\n    [train_sensor_summary, test_sensor_summary],\n    axis=0\n).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:28.912554Z","iopub.execute_input":"2025-08-11T13:49:28.912889Z","iopub.status.idle":"2025-08-11T13:49:35.201453Z","shell.execute_reply.started":"2025-08-11T13:49:28.912861Z","shell.execute_reply":"2025-08-11T13:49:35.200588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# (1) Sáp nhập train_demographics vào train_df nếu chưa thực hiện\ntrain_df = train_df.merge(\n    train_dem_df,\n    on=\"subject\",\n    how=\"left\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:35.202391Z","iopub.execute_input":"2025-08-11T13:49:35.202661Z","iopub.status.idle":"2025-08-11T13:49:35.867837Z","shell.execute_reply.started":"2025-08-11T13:49:35.202634Z","shell.execute_reply":"2025-08-11T13:49:35.867090Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Xử lý missing values","metadata":{}},{"cell_type":"markdown","source":"### 1. IMU – Gia tốc/quaternion (acc_*, rot_w/x/y/z)","metadata":{}},{"cell_type":"code","source":"def fix_imu(df):\n    imu_cols = [c for c in df.columns if c.startswith((\"acc_\", \"rot_\"))]\n    def _seq_fix(g):\n        g[imu_cols] = g[imu_cols].interpolate(limit_direction=\"both\")\n        # renormalize quaternion\n        q = g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]].to_numpy(float)\n        n = (q**2).sum(1)**0.5\n        n[n==0] = 1.0\n        q = q / n[:,None]\n        g[[\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]] = q\n        return g\n    return df.groupby(\"sequence_id\", group_keys=False).apply(_seq_fix)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:35.868807Z","iopub.execute_input":"2025-08-11T13:49:35.869107Z","iopub.status.idle":"2025-08-11T13:49:35.875792Z","shell.execute_reply.started":"2025-08-11T13:49:35.869079Z","shell.execute_reply":"2025-08-11T13:49:35.874659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. Thermopile (thm_1..thm_5)","metadata":{}},{"cell_type":"code","source":"def fix_thm(df):\n    thm = [f\"thm_{i}\" for i in range(1,6)]\n    return df.groupby(\"sequence_id\", group_keys=False)[thm].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:35.876855Z","iopub.execute_input":"2025-08-11T13:49:35.877232Z","iopub.status.idle":"2025-08-11T13:49:35.899925Z","shell.execute_reply.started":"2025-08-11T13:49:35.877202Z","shell.execute_reply":"2025-08-11T13:49:35.898805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. ToF – 5 cảm biến × 64 pixel (tof_i_vj)","metadata":{}},{"cell_type":"code","source":"def fix_tof_and_reduce(df):\n    # 1) sentinel -> NaN + flag\n    tof_pix = [c for c in df.columns if c.startswith(\"tof_\") and \"_v\" in c]\n    df[tof_pix] = df[tof_pix].replace(-1, np.nan)\n    # optional flags per sensor\n    for i in range(1,6):\n        cols = [f\"tof_{i}_v{p}\" for p in range(64) if f\"tof_{i}_v{p}\" in df]\n        df[f\"tof_{i}_missing_frac\"] = df[cols].isna().mean(axis=1)\n        df[f\"tof_{i}_mean\"] = df[cols].mean(axis=1)  # dimension reduction\n    # 2) interpolate the 5 means\n    mean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\n    df[mean_cols] = df.groupby(\"sequence_id\", group_keys=False)[mean_cols].apply(\n        lambda g: g.interpolate(limit_direction=\"both\").ffill().bfill()\n    )\n    # 3) drop heavy pixel columns to save RAM\n    df.drop(columns=tof_pix, inplace=True)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:35.903792Z","iopub.execute_input":"2025-08-11T13:49:35.904156Z","iopub.status.idle":"2025-08-11T13:49:35.919670Z","shell.execute_reply.started":"2025-08-11T13:49:35.904122Z","shell.execute_reply":"2025-08-11T13:49:35.918563Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 4. Pha/nhãn & metadata","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle_gesture = LabelEncoder().fit(train_df[\"gesture\"].astype(str))\ntrain_df[\"gesture_id\"] = le_gesture.transform(train_df[\"gesture\"].astype(str))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:35.920510Z","iopub.execute_input":"2025-08-11T13:49:35.920821Z","iopub.status.idle":"2025-08-11T13:49:36.064394Z","shell.execute_reply.started":"2025-08-11T13:49:35.920794Z","shell.execute_reply":"2025-08-11T13:49:36.063504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5. Pipeline tiền xử lý","metadata":{}},{"cell_type":"code","source":"def minimal_preprocess(df):\n    df = df.copy()\n    # 1) IMU\n    df = fix_imu(df)\n    # 2) Thermopile\n    thm_fixed = fix_thm(df)  # returns only thm cols\n    df[thm_fixed.columns] = thm_fixed\n    # 3) ToF -> means + flags, drop pixels\n    df = fix_tof_and_reduce(df)\n    # 4) Feature cơ bản (ví dụ):\n    df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n    df[\"rot_angle\"] = 2*np.arccos(np.clip(df[\"rot_w\"], -1, 1))\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.065214Z","iopub.execute_input":"2025-08-11T13:49:36.065506Z","iopub.status.idle":"2025-08-11T13:49:36.071375Z","shell.execute_reply.started":"2025-08-11T13:49:36.065480Z","shell.execute_reply":"2025-08-11T13:49:36.070381Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Cấu hình tham số và thư viện cần thiết:","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport joblib\nfrom typing import Tuple, List, Optional\nimport warnings\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom lightgbm import LGBMClassifier, log_evaluation, early_stopping\nfrom scipy.spatial.transform import Rotation as R\nimport kaggle_evaluation.cmi_inference_server\n\nwarnings.filterwarnings('ignore')\n\n# =============================================================================\n# CONFIGURATION\n# =============================================================================\n\nclass Config:\n    \"\"\"Central configuration class for training and data parameters\"\"\"\n    \n    # Paths for Kaggle environment\n    TRAIN_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\n    TRAIN_DEMOGRAPHICS_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\"\n    TEST_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\"\n    TEST_DEMOGRAPHICS_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\"\n    \n    # Training parameters\n    SEED = 42\n    N_FOLDS = 5\n    \n    # Feature columns\n    ACC_COLS = ['acc_x', 'acc_y', 'acc_z']\n    ROT_COLS = ['rot_w', 'rot_x', 'rot_y', 'rot_z']\n    \n    # LightGBM hyperparameters\n    LGBM_PARAMS = {\n        'objective': 'multiclass',  # Task type\n        'num_class': 18,            # Number of classes for multi-class classification\n        'n_estimators': 1024,       # Number of trees\n        'max_depth': 8,             # Maximum depth of trees\n        'learning_rate': 0.025,     # Learning rate\n        'colsample_bytree': 0.5,    # Fraction of features to use for each tree\n        'n_jobs': -1,               # Number of parallel threads\n        'num_leaves': 20,           # Number of leaves per tree\n        'random_state': 42,         # Random seed for reproducibility\n        'reg_alpha': 0.1,           # L1 regularization\n        'reg_lambda': 0.1,          # L2 regularization\n        'subsample': 0.5,           # Fraction of samples for each tree\n        'verbosity': -1,            # Suppress output verbosity\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.072129Z","iopub.execute_input":"2025-08-11T13:49:36.072375Z","iopub.status.idle":"2025-08-11T13:49:36.094985Z","shell.execute_reply.started":"2025-08-11T13:49:36.072356Z","shell.execute_reply":"2025-08-11T13:49:36.093817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# GESTURE MAPPING\n# =============================================================================\n\n# Gesture mapping (targets 0-7 are BFRB, 8-17 are non-BFRB)\nGESTURE_MAPPER = {\n    \"Above ear - pull hair\": 0,\n    \"Cheek - pinch skin\": 1,\n    \"Eyebrow - pull hair\": 2,\n    \"Eyelash - pull hair\": 3, \n    \"Forehead - pull hairline\": 4,\n    \"Forehead - scratch\": 5,\n    \"Neck - pinch skin\": 6, \n    \"Neck - scratch\": 7,\n    \n    \"Drink from bottle/cup\": 8,\n    \"Feel around in tray and pull out an object\": 9,\n    \"Glasses on/off\": 10,\n    \"Pinch knee/leg skin\": 11, \n    \"Pull air toward your face\": 12,\n    \"Scratch knee/leg skin\": 13,\n    \"Text on phone\": 14,\n    \"Wave hello\": 15,\n    \"Write name in air\": 16,\n    \"Write name on leg\": 17,\n}\n\nREVERSE_GESTURE_MAPPER = {v: k for k, v in GESTURE_MAPPER.items()}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.095892Z","iopub.execute_input":"2025-08-11T13:49:36.096280Z","iopub.status.idle":"2025-08-11T13:49:36.116400Z","shell.execute_reply.started":"2025-08-11T13:49:36.096250Z","shell.execute_reply":"2025-08-11T13:49:36.115468Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Cấu trúc mô hình và huấn luyện:","metadata":{}},{"cell_type":"code","source":"def check_gpu_availability():\n    \"\"\"Check if GPU is available for LightGBM\"\"\"\n    try:\n        # Try to create a simple LightGBM model with GPU\n        from lightgbm import LGBMClassifier\n        import numpy as np\n        \n        # Create dummy data\n        X_dummy = np.random.rand(100, 10)\n        y_dummy = np.random.randint(0, 2, 100)\n        \n        # Try GPU\n        model = LGBMClassifier(n_estimators=1, device='gpu', verbosity=-1)\n        model.fit(X_dummy, y_dummy)\n        print(\"✓ GPU available for LightGBM\")\n        return 'gpu'\n    except:\n        print(\"⚠️ GPU not available, using CPU for LightGBM\")\n        return 'cpu'\n\n# Initialize DEVICE based on the availability of GPU\nDEVICE = check_gpu_availability()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.117296Z","iopub.execute_input":"2025-08-11T13:49:36.117636Z","iopub.status.idle":"2025-08-11T13:49:36.158759Z","shell.execute_reply.started":"2025-08-11T13:49:36.117608Z","shell.execute_reply":"2025-08-11T13:49:36.157859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# MODEL BUILDING\n# =============================================================================\n\ndef build_lightgbm_model():\n    \"\"\"Build and return a LightGBM model based on configuration\"\"\"\n    model = LGBMClassifier(**Config.LGBM_PARAMS)\n    return model\n\n# =============================================================================\n# MODEL TRAINING\n# =============================================================================\n\nfrom sklearn.metrics import f1_score, accuracy_score\n\ndef train_lightgbm_models(X_train, y_train, subjects):\n    \"\"\"Train LightGBM models using stratified group k-fold cross-validation\"\"\"\n    \n    print(f\"Training LightGBM models with {Config.N_FOLDS}-fold cross-validation...\")\n    \n    # Prepare features (remove sequence_id)\n    feature_cols = [col for col in X_train.columns if col != 'sequence_id']\n    X_features = X_train[feature_cols]\n    \n    # Setup cross-validation\n    cv = StratifiedGroupKFold(n_splits=Config.N_FOLDS, shuffle=True, random_state=Config.SEED)\n    \n    models = []\n    oof_predictions = np.zeros(len(y_train))\n    cv_scores = []\n    \n    print(f\"Feature columns: {len(feature_cols)}\")\n    print(\"Starting cross-validation...\")\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X_features, y_train, subjects)):\n        print(f\"\\n--- Fold {fold + 1}/{Config.N_FOLDS} ---\")\n        \n        # Split data\n        X_fold_train = X_features.iloc[train_idx]\n        X_fold_val = X_features.iloc[val_idx]\n        y_fold_train = y_train[train_idx]\n        y_fold_val = y_train[val_idx]\n        \n        print(f\"Train size: {len(X_fold_train)}, Val size: {len(X_fold_val)}\")\n        \n        # Train model with monitoring and device detection\n        lgbm_params = Config.LGBM_PARAMS.copy()\n        lgbm_params['device'] = DEVICE  # Use detected device\n        \n        model = LGBMClassifier(**lgbm_params)\n        \n        print(f\"Training fold {fold + 1} with monitoring every 5 rounds (Device: {DEVICE})...\")\n        model.fit(\n            X_fold_train, y_fold_train,\n            eval_set=[(X_fold_train, y_fold_train), (X_fold_val, y_fold_val)],\n            eval_names=['train', 'valid'],\n            eval_metric=['multi_logloss', 'multi_error'],\n            callbacks=[\n                log_evaluation(period=5),  # Output every 5 rounds\n                early_stopping(stopping_rounds=100, verbose=True)\n            ]\n        )\n        \n        # Predictions\n        val_preds = model.predict(X_fold_val)\n        oof_predictions[val_idx] = val_preds\n        \n        # Calculate metrics\n        score, binary_f1, macro_f1 = competition_metric(y_fold_val, val_preds)\n        cv_scores.append(score)\n        \n        print(f\"Fold {fold + 1} - Competition Score: {score:.4f} (Binary F1: {binary_f1:.4f}, Macro F1: {macro_f1:.4f})\")\n        \n        models.append(model)\n    \n    # Overall CV performance\n    overall_score, overall_binary_f1, overall_macro_f1 = competition_metric(y_train, oof_predictions)\n    \n    print(f\"\\n{'='*60}\")\n    print(\"CROSS-VALIDATION RESULTS\")\n    print(f\"{'='*60}\")\n    print(f\"Overall Competition Score: {overall_score:.4f} ± {np.std(cv_scores):.4f}\")\n    print(f\"Overall Binary F1: {overall_binary_f1:.4f}\")\n    print(f\"Overall Macro F1: {overall_macro_f1:.4f}\")\n    print(f\"Fold scores: {['{:.4f}'.format(score) for score in cv_scores]}\")\n    print(f\"{'='*60}\")\n    \n    # Print final confirmation message\n    print(f\"\\n✓ Training completed successfully!\")\n    print(f\"✓ Final CV Score: {overall_score:.4f}\")\n    print(f\"✓ Core innovation: World acceleration transformation\")\n    print(f\"✓ Models ready for inference\")\n    \n    return models, cv_scores, overall_score  # Return all three values\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.159700Z","iopub.execute_input":"2025-08-11T13:49:36.159964Z","iopub.status.idle":"2025-08-11T13:49:36.175562Z","shell.execute_reply.started":"2025-08-11T13:49:36.159945Z","shell.execute_reply":"2025-08-11T13:49:36.174500Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Dự đoán và Inference:","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# PREDICTION FUNCTION\n# =============================================================================\n\ndef create_prediction_function(models, feature_cols, imu_cols):\n    \"\"\"Create a function for generating predictions using trained models\"\"\"\n    \n    def predict(sequence: pl.DataFrame, demographics: pl.DataFrame):\n        \"\"\"\n        Prediction function for Kaggle evaluation\n        Uses ensemble of trained LightGBM models\n        \"\"\"\n        try:\n            # Filter sequence to only include IMU columns that we trained with\n            available_cols = sequence.columns\n            sequence_imu_cols = [col for col in imu_cols if col in available_cols]\n            sequence_filtered = sequence.select(pl.col(sequence_imu_cols))\n            \n            # Extract features using the same method as training\n            features = extract_comprehensive_features(sequence_filtered, demographics)\n            \n            # Ensure we have the same features as training\n            missing_features = [col for col in feature_cols if col not in features.columns]\n            if missing_features:\n                print(f\"Warning: Missing features {missing_features}, filling with zeros\")\n                for col in missing_features:\n                    features[col] = 0\n            \n            X_pred = features[feature_cols]\n            \n            # Get predictions from all models\n            predictions = []\n            for model in models:\n                pred_probs = model.predict_proba(X_pred)\n                pred_class = np.argmax(pred_probs, axis=1)[0]\n                predictions.append(pred_class)\n            \n            # Ensemble prediction (majority vote)\n            final_prediction = max(set(predictions), key=predictions.count)\n            \n            # Convert back to gesture name\n            gesture_name = REVERSE_GESTURE_MAPPER[final_prediction]\n            \n            print(f\"Predicted: {gesture_name} (class {final_prediction})\")\n            return gesture_name\n            \n        except Exception as e:\n            print(f\"Prediction error: {e}\")\n            import traceback\n            traceback.print_exc()\n            return 'Text on phone'  # Fallback prediction\n\n    return predict\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.176578Z","iopub.execute_input":"2025-08-11T13:49:36.176951Z","iopub.status.idle":"2025-08-11T13:49:36.200745Z","shell.execute_reply.started":"2025-08-11T13:49:36.176922Z","shell.execute_reply":"2025-08-11T13:49:36.199524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Hàm Tính toán World Acceleration","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# CORE FEATURE ENGINEERING: WORLD ACCELERATION\n# =============================================================================\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    \"\"\"\n    Convert acceleration from device coordinates to world coordinates\n    \"\"\"\n    try:\n        # Convert quaternion format from [w, x, y, z] to [x, y, z, w] for scipy\n        rot_scipy = rot[:, [1, 2, 3, 0]]\n        \n        # Verify quaternions are valid (non-zero norm)\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        if np.any(norms < 1e-8):\n            rot_scipy[norms < 1e-8] = [0.0, 0.0, 0.0, 1.0]  # Identity quaternion in scipy format\n        \n        # Create rotation object and apply transformation\n        r = R.from_quat(rot_scipy)\n        acc_world = r.apply(acc)\n        \n    except Exception:\n        # Fallback to original acceleration if transformation fails\n        acc_world = acc.copy()\n    \n    return acc_world","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.201931Z","iopub.execute_input":"2025-08-11T13:49:36.202565Z","iopub.status.idle":"2025-08-11T13:49:36.224608Z","shell.execute_reply.started":"2025-08-11T13:49:36.202532Z","shell.execute_reply":"2025-08-11T13:49:36.223534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Features","metadata":{}},{"cell_type":"code","source":"def extract_statistical_features(data: np.ndarray, prefix: str) -> dict:\n    \"\"\"\n    Extract comprehensive statistical features from a 1D time series.\n    This function computes common statistical features like mean, std, skewness, kurtosis, etc.\n    \n    Returns features that capture:\n    - Central tendency: mean, median, mode region\n    - Spread: std, variance, range, IQR\n    - Shape: skewness, kurtosis\n    - Dynamics: differences, trends, changes\n    - Segments: beginning vs middle vs end behavior\n    \"\"\"\n    \n    features = {}\n    \n    # Basic statistics\n    features[f'{prefix}_mean'] = np.mean(data)\n    features[f'{prefix}_std'] = np.std(data)\n    features[f'{prefix}_var'] = np.var(data)\n    features[f'{prefix}_min'] = np.min(data)\n    features[f'{prefix}_max'] = np.max(data)\n    features[f'{prefix}_median'] = np.median(data)\n    features[f'{prefix}_q25'] = np.percentile(data, 25)\n    features[f'{prefix}_q75'] = np.percentile(data, 75)\n    features[f'{prefix}_iqr'] = np.percentile(data, 75) - np.percentile(data, 25)\n    \n    # Range and boundary features\n    features[f'{prefix}_range'] = np.max(data) - np.min(data)\n    features[f'{prefix}_first'] = data[0] if len(data) > 0 else 0\n    features[f'{prefix}_last'] = data[-1] if len(data) > 0 else 0\n    features[f'{prefix}_delta'] = data[-1] - data[0] if len(data) > 0 else 0\n    \n    # Higher order moments (shape of distribution)\n    if len(data) > 1 and np.std(data) > 1e-8:\n        features[f'{prefix}_skew'] = pd.Series(data).skew()\n        features[f'{prefix}_kurt'] = pd.Series(data).kurtosis()\n    else:\n        features[f'{prefix}_skew'] = 0\n        features[f'{prefix}_kurt'] = 0\n    \n    # Differential features (capture dynamics)\n    if len(data) > 1:\n        diff_data = np.diff(data)\n        features[f'{prefix}_diff_mean'] = np.mean(diff_data)\n        features[f'{prefix}_diff_std'] = np.std(diff_data)\n        features[f'{prefix}_n_changes'] = np.sum(np.abs(diff_data) > np.std(data) * 0.1)  # Significant changes\n    else:\n        features[f'{prefix}_diff_mean'] = 0\n        features[f'{prefix}_diff_std'] = 0\n        features[f'{prefix}_n_changes'] = 0\n    \n    # Correlation with time (trend detection)\n    if len(data) > 2:\n        time_indices = np.arange(len(data))\n        try:\n            corr_coef = np.corrcoef(time_indices, data)[0, 1]\n            features[f'{prefix}_time_corr'] = corr_coef if not np.isnan(corr_coef) else 0\n        except:\n            features[f'{prefix}_time_corr'] = 0\n    else:\n        features[f'{prefix}_time_corr'] = 0\n    \n    # Segment features (beginning, middle, end patterns)\n    seq_len = len(data)\n    if seq_len >= 9:  # Need sufficient data for meaningful segments\n        seg_size = seq_len // 3\n        seg1 = data[:seg_size]           # Beginning (Transition phase)\n        seg2 = data[seg_size:2*seg_size] # Middle (Pause phase)  \n        seg3 = data[2*seg_size:]         # End (Gesture phase)\n        \n        features[f'{prefix}_seg1_mean'] = np.mean(seg1)\n        features[f'{prefix}_seg2_mean'] = np.mean(seg2)\n        features[f'{prefix}_seg3_mean'] = np.mean(seg3)\n        \n        features[f'{prefix}_seg1_std'] = np.std(seg1)\n        features[f'{prefix}_seg2_std'] = np.std(seg2)\n        features[f'{prefix}_seg3_std'] = np.std(seg3)\n        \n        # Segment transitions (important for distinguishing gesture types)\n        features[f'{prefix}_seg1_to_seg2'] = np.mean(seg2) - np.mean(seg1)\n        features[f'{prefix}_seg2_to_seg3'] = np.mean(seg3) - np.mean(seg2)\n    else:\n        # Not enough data for meaningful segments\n        for seg in [1, 2, 3]:\n            features[f'{prefix}_seg{seg}_mean'] = features[f'{prefix}_mean']\n            features[f'{prefix}_seg{seg}_std'] = features[f'{prefix}_std']\n        features[f'{prefix}_seg1_to_seg2'] = 0\n        features[f'{prefix}_seg2_to_seg3'] = 0\n    \n    return features\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.225643Z","iopub.execute_input":"2025-08-11T13:49:36.225881Z","iopub.status.idle":"2025-08-11T13:49:36.252568Z","shell.execute_reply.started":"2025-08-11T13:49:36.225863Z","shell.execute_reply":"2025-08-11T13:49:36.251430Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef competition_metric(y_true, y_pred) -> tuple:\n    \"\"\"Calculate the competition metric (Binary F1 + Macro F1) / 2\"\"\"\n    \n    # Binary F1: BFRB vs non-BFRB (classifying BFRB gestures as 1, others as 0)\n    binary_f1 = f1_score(\n        np.where(y_true <= 7, 1, 0),\n        np.where(y_pred <= 7, 1, 0),\n        zero_division=0.0,\n    )\n    \n    # Macro F1: Calculating F1 score across all BFRB gestures (0-7)\n    macro_f1 = f1_score(\n        np.where(y_true <= 7, y_true, 99),  # Map non-BFRB gestures to class 99\n        np.where(y_pred <= 7, y_pred, 99),  # Map non-BFRB gestures to class 99\n        average=\"macro\", \n        zero_division=0.0,\n    )\n    \n    # Final competition score: average of binary F1 and macro F1\n    final_score = 0.5 * (binary_f1 + macro_f1)\n    \n    return final_score, binary_f1, macro_f1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.253668Z","iopub.execute_input":"2025-08-11T13:49:36.253966Z","iopub.status.idle":"2025-08-11T13:49:36.276605Z","shell.execute_reply.started":"2025-08-11T13:49:36.253942Z","shell.execute_reply":"2025-08-11T13:49:36.275636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    \"\"\"\n    Handle missing values in quaternion data intelligently\n    \n    Key insight: Quaternions must have unit length |q| = 1\n    If one component is missing, we can reconstruct it from the others\n    \"\"\"\n    rot_cleaned = rot_data.copy()\n    \n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        \n        if missing_count == 0:\n            # No missing values, normalize to unit quaternion\n            norm = np.linalg.norm(row)\n            if norm > 1e-8:\n                rot_cleaned[i] = row / norm\n            else:\n                rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]  # Identity quaternion\n                \n        elif missing_count == 1:\n            # One missing value, reconstruct using unit quaternion constraint\n            # |w|² + |x|² + |y|² + |z|² = 1\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            \n            sum_squares = np.sum(valid_values**2)\n            if sum_squares <= 1.0:\n                missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n                # Choose sign for continuity with previous quaternion\n                if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n                    if rot_cleaned[i-1, missing_idx] < 0:\n                        missing_value = -missing_value\n                rot_cleaned[i, missing_idx] = missing_value\n                rot_cleaned[i, ~np.isnan(row)] = valid_values\n            else:\n                rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n        else:\n            # More than one missing value, use identity quaternion\n            rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n    \n    return rot_cleaned\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.277928Z","iopub.execute_input":"2025-08-11T13:49:36.278352Z","iopub.status.idle":"2025-08-11T13:49:36.300569Z","shell.execute_reply.started":"2025-08-11T13:49:36.278321Z","shell.execute_reply":"2025-08-11T13:49:36.299564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# SIMPLIFIED FEATURE EXTRACTION\n# =============================================================================\n\ndef extract_comprehensive_features(sequence: pl.DataFrame, demographics: pl.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Extract features from IMU data with world acceleration transformation.\n    This function should return a DataFrame with extracted features.\n    \"\"\"\n    # Convert to pandas for processing\n    seq_df = sequence.to_pandas()\n    demo_df = demographics.to_pandas()\n    \n    # Handle missing values in basic sensor data\n    acc_data = seq_df[Config.ACC_COLS].copy()\n    acc_data = acc_data.ffill().bfill().fillna(0)\n    \n    rot_data = seq_df[Config.ROT_COLS].copy()\n    rot_data = rot_data.ffill().bfill()\n    \n    # Handle quaternion missing values and normalize\n    rot_data_clean = handle_quaternion_missing_values(rot_data.values)\n    \n    # CORE INNOVATION: Compute world acceleration\n    try:\n        world_acc_data = compute_world_acceleration(acc_data.values, rot_data_clean)\n    except Exception as e:\n        print(f\"Warning: World acceleration computation failed: {e}\")\n        world_acc_data = acc_data.values.copy()  # Fallback to device coordinates\n    \n    # Initialize feature dictionary\n    features = {}\n    \n    # Add sequence metadata\n    features['sequence_length'] = len(seq_df)\n    \n    # Add demographics features\n    if len(demo_df) > 0:\n        demo_row = demo_df.iloc[0]\n        features['age'] = demo_row.get('age', 0)\n        features['adult_child'] = demo_row.get('adult_child', 0)\n        features['sex'] = demo_row.get('sex', 0)\n        features['handedness'] = demo_row.get('handedness', 0)\n        features['height_cm'] = demo_row.get('height_cm', 0)\n        features['shoulder_to_wrist_cm'] = demo_row.get('shoulder_to_wrist_cm', 0)\n        features['elbow_to_wrist_cm'] = demo_row.get('elbow_to_wrist_cm', 0)\n    \n    # Define feature arrays for statistical extraction\n    feature_arrays = {\n        'acc': acc_data.values,           # Device acceleration (3D)\n        'rot': rot_data_clean,            # Rotation quaternion (4D) \n        'world_acc': world_acc_data,      # World acceleration (3D) - KEY INNOVATION\n    }\n    \n    # Extract statistical features for each data source\n    for source_name, array in feature_arrays.items():\n        if array.ndim == 1:\n            array = array.reshape(-1, 1)\n        \n        n_features = array.shape[1]\n        \n        for feat_idx in range(n_features):\n            feat_data = array[:, feat_idx]\n            \n            # Create feature name\n            if source_name == 'acc':\n                axis_names = ['x', 'y', 'z']\n                prefix = f\"acc_{axis_names[feat_idx]}\"\n            elif source_name == 'rot':\n                comp_names = ['w', 'x', 'y', 'z']\n                prefix = f\"rot_{comp_names[feat_idx]}\"\n            elif source_name == 'world_acc':\n                axis_names = ['x', 'y', 'z']  \n                prefix = f\"world_acc_{axis_names[feat_idx]}\"\n            else:\n                prefix = f\"{source_name}_{feat_idx}\" if n_features > 1 else source_name\n            \n            # Extract comprehensive statistical features\n            features.update(extract_statistical_features(feat_data, prefix))\n    \n    # Compute magnitude features (important for motion intensity)\n    acc_magnitude = np.linalg.norm(acc_data.values, axis=1)\n    world_acc_magnitude = np.linalg.norm(world_acc_data, axis=1)\n    \n    features.update(extract_statistical_features(acc_magnitude, 'acc_magnitude'))\n    features.update(extract_statistical_features(world_acc_magnitude, 'world_acc_magnitude'))\n    \n    # Cross-feature: difference between device and world acceleration magnitudes\n    acc_world_diff = acc_magnitude - world_acc_magnitude\n    features.update(extract_statistical_features(acc_world_diff, 'acc_world_diff'))\n    \n    # Convert to DataFrame\n    result_df = pd.DataFrame([features])\n    \n    # Handle any remaining NaN values\n    result_df = result_df.fillna(0)\n    \n    return result_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.301590Z","iopub.execute_input":"2025-08-11T13:49:36.301895Z","iopub.status.idle":"2025-08-11T13:49:36.328854Z","shell.execute_reply.started":"2025-08-11T13:49:36.301866Z","shell.execute_reply":"2025-08-11T13:49:36.328112Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Chuẩn bị và tải dữ liệu:","metadata":{}},{"cell_type":"code","source":"def load_and_prepare_data():\n    \"\"\"Load and prepare training data with comprehensive features\"\"\"\n    \n    print(\"Loading training data...\")\n    train_df = pl.read_csv(Config.TRAIN_PATH)\n    train_demographics = pl.read_csv(Config.TRAIN_DEMOGRAPHICS_PATH)\n    \n    print(\"Loading test data...\")\n    test_df = pl.read_csv(Config.TEST_PATH)\n    test_demographics = pl.read_csv(Config.TEST_DEMOGRAPHICS_PATH)\n    \n    # Get common columns between train and test (exclude thermal and ToF sensors)\n    train_cols = set(train_df.columns)\n    test_cols = set(test_df.columns)\n    common_cols = train_cols.intersection(test_cols)\n    \n    # Filter to IMU-only columns (remove thermal and ToF sensors)\n    imu_cols = [col for col in common_cols if not (col.startswith('thm_') or col.startswith('tof_'))]\n    \n    print(f\"✓ Using {len(imu_cols)} common IMU columns\")\n    print(f\"✓ Train-only columns: {train_cols - test_cols}\")\n    print(f\"✓ Test-only columns: {test_cols - train_cols}\")\n    \n    print(\"Extracting features for training sequences...\")\n    train_features_list = []\n    train_labels = []\n    train_subjects = []\n    train_sequence_ids = []\n    \n    # Group by sequence_id for training data - need to include gesture column for labels\n    train_imu_cols = imu_cols + ['gesture'] if 'gesture' not in imu_cols else imu_cols\n    train_sequences = train_df.select(pl.col(train_imu_cols)).group_by('sequence_id', maintain_order=True)\n    \n    for sequence_id, sequence_data in train_sequences:\n        # Get sequence features\n        sequence_id_val = sequence_id[0] if isinstance(sequence_id, tuple) else sequence_id\n        \n        # Get demographics for this sequence\n        subject_id = sequence_data['subject'][0]\n        subject_demographics = train_demographics.filter(pl.col('subject') == subject_id)\n        \n        # Extract features (only IMU columns for feature extraction)\n        imu_only_data = sequence_data.select(pl.col(imu_cols))\n        features = extract_comprehensive_features(imu_only_data, subject_demographics)\n        features['sequence_id'] = sequence_id_val\n        \n        train_features_list.append(features)\n        \n        # Get label (gesture) for this sequence\n        gesture = sequence_data['gesture'][0]\n        label = GESTURE_MAPPER[gesture]\n        train_labels.append(label)\n        train_subjects.append(subject_id)\n        train_sequence_ids.append(sequence_id_val)\n    \n    # Combine all training features\n    X_train = pd.concat(train_features_list, ignore_index=True)\n    y_train = np.array(train_labels)\n    subjects = np.array(train_subjects)\n\n    # Extract features for test sequences\n    print(\"Extracting features for test sequences...\")\n    test_features_list = []\n    test_sequence_ids = []\n    \n    # Group by sequence_id for test data  \n    test_sequences = test_df.select(pl.col(imu_cols)).group_by('sequence_id', maintain_order=True)\n    \n    for sequence_id, sequence_data in test_sequences:\n        sequence_id_val = sequence_id[0] if isinstance(sequence_id, tuple) else sequence_id\n        \n        # Get demographics for this sequence\n        subject_id = sequence_data['subject'][0]\n        subject_demographics = test_demographics.filter(pl.col('subject') == subject_id)\n        \n        # Extract features using the same function as training\n        features = extract_comprehensive_features(sequence_data, subject_demographics)\n        features['sequence_id'] = sequence_id_val\n        \n        test_features_list.append(features)\n        test_sequence_ids.append(sequence_id_val)\n    \n    # Combine all test features\n    X_test = pd.concat(test_features_list, ignore_index=True)\n    \n    print(f\"✓ Training features shape: {X_train.shape}\")\n    print(f\"✓ Training labels shape: {y_train.shape}\")\n    print(f\"✓ Test features shape: {X_test.shape}\")\n    print(f\"✓ Number of features: {X_train.shape[1] - 1}\")  # -1 for sequence_id\n    \n    # Now remove `sequence_id` from X_train and X_test\n    X_train = X_train.drop(columns=['sequence_id'], errors='ignore')\n    X_test = X_test.drop(columns=['sequence_id'], errors='ignore')\n    \n    # Return processed data\n    return X_train, y_train, subjects, X_test, test_sequence_ids, imu_cols\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.329835Z","iopub.execute_input":"2025-08-11T13:49:36.330098Z","iopub.status.idle":"2025-08-11T13:49:36.353006Z","shell.execute_reply.started":"2025-08-11T13:49:36.330073Z","shell.execute_reply":"2025-08-11T13:49:36.351982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference Server Setup:","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# INFERENCE SERVER SETUP\n# =============================================================================\n\ndef setup_inference_server(predict_function):\n    \"\"\"Set up the inference server to serve predictions\"\"\"\n    inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict_function)\n    \n    # If in competition rerun, use Kaggle environment to serve\n    if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n        inference_server.serve()\n    else:\n        inference_server.run_local_gateway(data_paths=(Config.TEST_PATH, Config.TEST_DEMOGRAPHICS_PATH))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.354366Z","iopub.execute_input":"2025-08-11T13:49:36.354731Z","iopub.status.idle":"2025-08-11T13:49:36.377632Z","shell.execute_reply.started":"2025-08-11T13:49:36.354700Z","shell.execute_reply":"2025-08-11T13:49:36.376534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Chạy Toàn Bộ Pipeline:","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# MAIN EXECUTION PIPELINE\n# =============================================================================\n\ndef main():\n    \"\"\"Main function for training, validation, and inference\"\"\"\n    \n    # Load data\n    X_train, y_train, subjects, X_test, test_sequence_ids, imu_cols = load_and_prepare_data()\n\n    # Train models\n    models, cv_scores, overall_score = train_lightgbm_models(X_train, y_train, subjects)\n\n    # Prepare prediction function\n    feature_cols = [col for col in X_train.columns if col != 'sequence_id']\n    predict_function = create_prediction_function(models, feature_cols, imu_cols)  # Khởi tạo predict_function\n\n    # Set up inference server\n    setup_inference_server(predict_function)\n\n    # Print final success message\n    print(\"\\n✓ Training completed successfully!\")\n    print(f\"✓ Final CV Score: {overall_score:.4f}\")\n    print(f\"✓ Models ready for inference\")\n    \n    # Return the required values\n    return predict_function, models, cv_scores\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.378652Z","iopub.execute_input":"2025-08-11T13:49:36.378929Z","iopub.status.idle":"2025-08-11T13:49:36.396819Z","shell.execute_reply.started":"2025-08-11T13:49:36.378895Z","shell.execute_reply":"2025-08-11T13:49:36.395581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Chạy Pipeline Chính","metadata":{}},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    predict_function, trained_models, cv_scores = main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T13:49:36.397861Z","iopub.execute_input":"2025-08-11T13:49:36.398179Z","iopub.status.idle":"2025-08-11T14:00:02.843941Z","shell.execute_reply.started":"2025-08-11T13:49:36.398148Z","shell.execute_reply":"2025-08-11T14:00:02.842205Z"}},"outputs":[],"execution_count":null}]}