{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:18.859249Z","iopub.execute_input":"2025-07-26T08:20:18.859597Z","iopub.status.idle":"2025-07-26T08:20:18.885504Z","shell.execute_reply.started":"2025-07-26T08:20:18.859574Z","shell.execute_reply":"2025-07-26T08:20:18.884235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"I. Nhập và hiểu dữ liệu","metadata":{}},{"cell_type":"markdown","source":"Bước 1: Tải và hiểu cấu trúc data\n\n- Tệp cần đọc:\n    - `train.csv`, `test.csv`: dữ liệu sensor theo thời gian.\n    - `train_demographics.csv`, `test_demographics.csv`: thông tin người dùng.\n    - `gesture_mapping.csv` (nếu có).\n- Dùng `polars` hoặc `pandas` để đọc, kiểm tra `.shape`, `.head()`, `.info()`.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:18.886985Z","iopub.execute_input":"2025-07-26T08:20:18.887223Z","iopub.status.idle":"2025-07-26T08:20:18.892608Z","shell.execute_reply.started":"2025-07-26T08:20:18.887203Z","shell.execute_reply":"2025-07-26T08:20:18.890878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the datasets\n#Dữ liệu cảm biến cho tập huấn luyện (Train Sensor Data): \ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\n#Thông tin nhân khẩu học của đối tượng trong tập huấn luyện (giới tính, tuổi, ...): \ntrain_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\n\n#Dữ liệu cảm biến cho tập kiểm tra:\ntest_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\n\n# Thông tin nhân khẩu học của đối tượng trong tập kiểm tra:\ntest_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:18.893849Z","iopub.execute_input":"2025-07-26T08:20:18.894188Z","iopub.status.idle":"2025-07-26T08:20:38.830067Z","shell.execute_reply.started":"2025-07-26T08:20:18.89416Z","shell.execute_reply":"2025-07-26T08:20:38.829214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Tổng hợp dữ liệu vào dictionary: Tạo dictionary datasets chứa 4 bộ dữ liệu chính, giúp dễ quản lý và xử lý lặp đi lặp lại.\ndatasets = {\n    \"Train Data\": train_df,\n    \"Train Demographics\": train_dem_df,\n    \"Test Data\": test_df,\n    \"Test Demographics\": test_dem_df,\n}\n\n# Print shapes\n#Giải thích: .items(): Duyệt qua từng cặp tên và DataFrame.\n#            .shape: Trả về tuple (số dòng, số cột).\n#            print(...): In ra tên dataset, số dòng và số cột, giúp bạn hình dung dữ liệu có bao nhiêu bản ghi và thuộc tính.\n\n\nfor name, df in datasets.items():\n    num_rows, num_cols = df.shape\n    print(f\"{name}:\")\n    print(f\"  Number of Rows: {num_rows}\")\n    print(f\"  Number of Columns: {num_cols}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:38.831656Z","iopub.execute_input":"2025-07-26T08:20:38.831863Z","iopub.status.idle":"2025-07-26T08:20:38.839815Z","shell.execute_reply.started":"2025-07-26T08:20:38.831849Z","shell.execute_reply":"2025-07-26T08:20:38.838737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra trùng lặp (Duplicate Rows): .duplicated(): Trả về Series boolean, dòng nào trùng lặp sẽ là True.\n#                                     .sum(): Đếm tổng số dòng trùng lặp.\n#Kiểm tra xem dữ liệu có bản ghi nào bị trùng không — quan trọng cho việc làm sạch dữ liệu.\n\n# Đếm các hàng trùng lặp trong train_df\ntrain_duplicates = train_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong  test_df\ntest_duplicates = test_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong train_dem_df (optional)\ntrain_dem_duplicates = train_dem_df.duplicated().sum()\n# Đếm các hàng trùng lặp trong test_dem_df (optional)\ntest_dem_duplicates = test_dem_df.duplicated().sum()\n\n# In số lượng dòng trùng:\nprint(f\"Number of duplicate rows in train_df: {train_duplicates}\")\nprint(f\"Number of duplicate rows in test_df: {test_duplicates}\")\nprint(f\"Number of duplicate rows in train_dem_df: {train_dem_duplicates}\")\nprint(f\"Number of duplicate rows in test_dem_df: {test_dem_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:38.840954Z","iopub.execute_input":"2025-07-26T08:20:38.841251Z","iopub.status.idle":"2025-07-26T08:20:42.749134Z","shell.execute_reply.started":"2025-07-26T08:20:38.841224Z","shell.execute_reply":"2025-07-26T08:20:42.747339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Hiển thị dữ liệu mẫu: .head(2): Xem trước 2 dòng đầu của mỗi bảng.\n#                      display(...): Dùng trong notebook để hiển thị bảng dữ liệu đẹp, rõ ràng.\n\ndisplay(train_df.head(2))\ndisplay(train_dem_df.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:42.750424Z","iopub.execute_input":"2025-07-26T08:20:42.750731Z","iopub.status.idle":"2025-07-26T08:20:42.774705Z","shell.execute_reply.started":"2025-07-26T08:20:42.750705Z","shell.execute_reply":"2025-07-26T08:20:42.773352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(test_df.head(2))\ndisplay(test_dem_df.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:42.775931Z","iopub.execute_input":"2025-07-26T08:20:42.776248Z","iopub.status.idle":"2025-07-26T08:20:42.809455Z","shell.execute_reply.started":"2025-07-26T08:20:42.776223Z","shell.execute_reply":"2025-07-26T08:20:42.808662Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Kiểm tra giá trị thiếu (NaN)","metadata":{}},{"cell_type":"code","source":"def check_missing_values(df, name=\"\"):\n    print(f\"🔍 Missing values in {name}\")\n    missing = df.isna().sum()\n    missing = missing[missing > 0]\n    if missing.empty:\n        print(\"✅ Không có giá trị thiếu.\")\n    else:\n        print(missing.sort_values(ascending=False))\n    print(\"\\n\")\n\ncheck_missing_values(train_df, \"train_df\")\ncheck_missing_values(test_df, \"test_df\")\ncheck_missing_values(train_dem_df, \"train_dem_df\")\ncheck_missing_values(test_dem_df, \"test_dem_df\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:42.810347Z","iopub.execute_input":"2025-07-26T08:20:42.810598Z","iopub.status.idle":"2025-07-26T08:20:43.224467Z","shell.execute_reply.started":"2025-07-26T08:20:42.81058Z","shell.execute_reply":"2025-07-26T08:20:43.223065Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Thống kê mô tả các cột số (sensor)","metadata":{}},{"cell_type":"code","source":"sensor_cols = [col for col in train_df.columns if col not in ['timestamp', 'participant_id', 'gesture', 'event']]\n\nprint(\"📈 Thống kê mô tả các cột sensor:\")\nprint(train_df[sensor_cols].describe().T)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:43.225443Z","iopub.execute_input":"2025-07-26T08:20:43.225665Z","iopub.status.idle":"2025-07-26T08:20:48.889651Z","shell.execute_reply.started":"2025-07-26T08:20:43.225648Z","shell.execute_reply":"2025-07-26T08:20:48.888566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Acceleration quá lớn (vượt ngưỡng vật lý)","metadata":{}},{"cell_type":"code","source":"# Giả sử ngưỡng vật lý là +/- 50 m/s^2\nacc_cols = [col for col in train_df.columns if \"acc\" in col]\n\nprint(\"⚠️ Giá trị acceleration vượt quá ±50 m/s²:\")\nfor col in acc_cols:\n    count = (train_df[col].abs() > 50).sum()\n    if count > 0:\n        print(f\"{col}: {count} giá trị bất thường\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:48.891815Z","iopub.execute_input":"2025-07-26T08:20:48.892071Z","iopub.status.idle":"2025-07-26T08:20:48.903839Z","shell.execute_reply.started":"2025-07-26T08:20:48.89205Z","shell.execute_reply":"2025-07-26T08:20:48.902784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tof_cols = [col for col in train_df.columns if \"tof\" in col]\n\nprint(\"⚠️ Giá trị ToF = 0 hoặc NaN:\")\nfor col in tof_cols:\n    zero_count = (train_df[col] == 0).sum()\n    nan_count = train_df[col].isna().sum()\n    if zero_count > 0 or nan_count > 0:\n        print(f\"{col}: {zero_count} giá trị = 0, {nan_count} NaN\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:48.904537Z","iopub.execute_input":"2025-07-26T08:20:48.904704Z","iopub.status.idle":"2025-07-26T08:20:49.444535Z","shell.execute_reply.started":"2025-07-26T08:20:48.904691Z","shell.execute_reply":"2025-07-26T08:20:49.443303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thermo_cols = [col for col in train_df.columns if \"thm\" in col]\n\nprint(\"🌡️ Nhiệt độ bất thường (<15°C hoặc >45°C):\")\nfor col in thermo_cols:\n    abnormal = ((train_df[col] < 15) | (train_df[col] > 45)).sum()\n    if abnormal > 0:\n        print(f\"{col}: {abnormal} giá trị bất thường\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:49.445816Z","iopub.execute_input":"2025-07-26T08:20:49.446045Z","iopub.status.idle":"2025-07-26T08:20:49.466172Z","shell.execute_reply.started":"2025-07-26T08:20:49.446027Z","shell.execute_reply":"2025-07-26T08:20:49.465408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Chuẩn bị hàm thống kê mô tả có lọc cột: \n# excluded_prefixes: Bộ tiền tố để loại các cột cảm biến như: 'acc_': Gia tốc kế, 'rot_': Cảm biến xoay, 'thm_': Nhiệt độ/Độ ẩm. 'tof_': Time-of-Flight (cảm biến khoảng cách).\n\n# Define excluded prefixes\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\n\n#Hàm filtered_describe: Lọc các cột không thuộc cảm biến và có kiểu số.\n#                       Dùng .describe().T để thống kê mô tả (mean, std, min, max,...) và chuyển vị bảng cho dễ nhìn.\n#                       display: Hiển thị bảng thống kê.\n\ndef filtered_describe(df, name):\n    # Exclude sensor columns\n    filtered_cols = [col for col in df.columns \n                     if not col.startswith(excluded_prefixes) and pd.api.types.is_numeric_dtype(df[col])]\n    \n    # Describe and style\n    print(f'\\n➡️ Description of numerical columns in {name}')\n    return df[filtered_cols].describe().T.style.background_gradient(cmap='viridis')\n\n# Only analyze train sets!\ndisplay(filtered_describe(train_df, \"train_df\"))\ndisplay(filtered_describe(train_dem_df, \"train_dem_df\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:49.467114Z","iopub.execute_input":"2025-07-26T08:20:49.467417Z","iopub.status.idle":"2025-07-26T08:20:49.519029Z","shell.execute_reply.started":"2025-07-26T08:20:49.467395Z","shell.execute_reply":"2025-07-26T08:20:49.518172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra thiếu giá trị & thống kê cảm biến: \n#Danh sách các cột không thuộc cảm biến (có thể là ID, thông tin khác). \n\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nsensor_cols = [col for col in train_df.columns if not col.startswith(excluded_prefixes)]\n\n# Sensor Data Summary for TRAIN\n#isnull().sum(): Đếm số giá trị bị thiếu.\nmissing_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    '[TRAIN] Missing Count': train_df[sensor_cols].isnull().sum().values,\n    '[TRAIN] Missing %': (train_df[sensor_cols].isnull().sum().values / len(train_df)) * 100\n})\n\n#nunique(): Đếm số lượng giá trị duy nhất.\nunique_sensor_train = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Unique Values [TRAIN]': train_df[sensor_cols].nunique().values\n})\n\n#dtypes: Lấy kiểu dữ liệu của từng cột.\ndtypes_sensor = pd.DataFrame({\n    'Feature': sensor_cols,\n    'Data Type': train_df[sensor_cols].dtypes.values\n})\n\n# Merge all summaries (NO test set)\n#merge: Gộp các bảng thống kê thành bảng duy nhất theo Feature.\nsensor_summary = missing_sensor_train \\\n    .merge(unique_sensor_train, on='Feature', how='left') \\\n    .merge(dtypes_sensor, on='Feature', how='left')\n\n# Display styled DataFrame (mask NaNs just for styling)\n#fillna(0): Điền giá trị thiếu bằng 0 (cho đẹp mắt khi hiển thị).\n#.style.background_gradient: Tô màu nền theo giá trị giúp dễ nhìn.\nstyled_df = sensor_summary.fillna(0)\nstyled_df.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:49.519887Z","iopub.execute_input":"2025-07-26T08:20:49.520599Z","iopub.status.idle":"2025-07-26T08:20:50.253728Z","shell.execute_reply.started":"2025-07-26T08:20:49.520555Z","shell.execute_reply":"2025-07-26T08:20:50.252502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Thống kê tương tự cho nhân khẩu học: Tương tự như phần thống kê cảm biến nhưng áp dụng cho dữ liệu nhân khẩu học.\n\n# Cột nhân khẩu học (không loại trừ)\ndem_cols = train_dem_df.columns\n\n# Giá trị bị thiếu trong dữ liệu nhân khẩu học của train \nmissing_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    '[TRAIN DEMO] Missing Count': train_dem_df[dem_cols].isnull().sum().values,\n    '[TRAIN DEMO] Missing %': (train_dem_df[dem_cols].isnull().sum().values / len(train_dem_df)) * 100\n})\n\n# Giá trị duy nhất được tính trong dữ liệu nhân khẩu học của train \nunique_demo_train = pd.DataFrame({\n    'Feature': dem_cols,\n    'Unique Values [TRAIN DEMO]': train_dem_df[dem_cols].nunique().values\n})\n\n# Data types\ndtypes_demo = pd.DataFrame({\n    'Feature': dem_cols,\n    'Data Type': train_dem_df[dem_cols].dtypes.values\n})\n\n# Merge summaries (train only)\ndemo_summary = (\n    missing_demo_train\n    .merge(unique_demo_train, on='Feature', how='left')\n    .merge(dtypes_demo, on='Feature', how='left')\n)\n\n# Display styled summary\ndemo_summary.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:50.254782Z","iopub.execute_input":"2025-07-26T08:20:50.255095Z","iopub.status.idle":"2025-07-26T08:20:50.282808Z","shell.execute_reply.started":"2025-07-26T08:20:50.255065Z","shell.execute_reply":"2025-07-26T08:20:50.28144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Sensor column groups (use train_df only)\n# Phân nhóm các cột cảm biến:\nacc_cols = [col for col in train_df.columns if col.startswith('acc_')]\nrot_cols = [col for col in train_df.columns if col.startswith('rot_')]\nthm_cols = [col for col in train_df.columns if col.startswith('thm_')]\ntof_cols = [col for col in train_df.columns if col.startswith('tof_')]\n\n# Helper function for summary stats (train only)\n#Hàm thống kê chi tiết từng nhóm cảm biến (Tỷ lệ thiếu giá trị; Giá trị nhỏ nhất, lớn nhất; Trung bình, độ lệch chuẩn.)\ndef sensor_summary(df, cols, name, dataset_name):\n    summary = pd.DataFrame({\n        'Feature': cols,\n        f'{dataset_name} Missing %': df[cols].isnull().mean().values * 100,\n        f'{dataset_name} Min': df[cols].min().values,\n        f'{dataset_name} Max': df[cols].max().values,\n        f'{dataset_name} Mean': df[cols].mean().values,\n        f'{dataset_name} Std': df[cols].std().values\n    })\n    summary.insert(0, 'Sensor', name)\n    return summary\n\n# Compute train summary for each sensor type\n#Thống kê tổng hợp cho tập train: Duyệt từng nhóm cảm biến và gom kết quả thống kê lại.\n# pd.concat: Ghép bảng lại thành bảng thống kê tổng hợp.\ndef train_sensor_summary(train_df, sensor_cols_dict):\n    all_train = []\n    for sensor_name, cols in sensor_cols_dict.items():\n        all_train.append(sensor_summary(train_df, cols, sensor_name, 'Train'))\n    train_summary = pd.concat(all_train, ignore_index=True)\n    return train_summary\n\n# Sensor groups to process\nsensor_cols_dict = {\n    'acc': acc_cols,\n    'rot': rot_cols,\n    'thm': thm_cols,\n    'tof': tof_cols\n}\n\n# Run train summary only\nsensor_train_summary = train_sensor_summary(train_df, sensor_cols_dict)\n\n#rung bình theo nhóm cảm biến: \n#                             .groupby('Sensor'): Gom nhóm theo loại cảm biến.\n#                             .mean(numeric_only=True): Trung bình các chỉ số số học.\n#                             Tô màu bảng kết quả giúp dễ phân tích.\n\n# Show sensor-level averages for train\nsummary_by_group = sensor_train_summary.groupby('Sensor').mean(numeric_only=True)\n\n# Display styled table\nsummary_by_group.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:20:50.283714Z","iopub.execute_input":"2025-07-26T08:20:50.283946Z","iopub.status.idle":"2025-07-26T08:20:53.043774Z","shell.execute_reply.started":"2025-07-26T08:20:50.283926Z","shell.execute_reply":"2025-07-26T08:20:53.042849Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Phân tích nhãn (Target - gesture)","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# ✅ Biểu đồ phân phối nhãn (gesture)\nplt.figure(figsize=(12, 6))\nsns.countplot(x='gesture', data=train_df, order=train_df['gesture'].value_counts().index)\nplt.title(\"Phân phối nhãn (gesture) trong tập train\")\nplt.xlabel(\"Gesture\")\nplt.ylabel(\"Số lượng\")\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n\n# ✅ Số lượng gesture khác nhau mỗi subject thực hiện\ngesture_per_user = train_df.groupby('subject')['gesture'].nunique()\nprint(\"\\n➡️ Thống kê số lượng gesture khác nhau mỗi subject thực hiện:\")\nprint(gesture_per_user.describe())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:22:59.352357Z","iopub.execute_input":"2025-07-26T08:22:59.352652Z","iopub.status.idle":"2025-07-26T08:22:59.797454Z","shell.execute_reply.started":"2025-07-26T08:22:59.352633Z","shell.execute_reply":"2025-07-26T08:22:59.796493Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Phân tích theo người dùng (Group by participant)","metadata":{}},{"cell_type":"code","source":"# Số dòng dữ liệu mỗi người dùng (theo cột 'subject')\nuser_counts = train_df['subject'].value_counts()\n\nplt.figure(figsize=(10, 6))\nsns.histplot(user_counts, bins=30, kde=True)\nplt.title(\"Số dòng dữ liệu mỗi người dùng\")\nplt.xlabel(\"Số dòng\")\nplt.ylabel(\"Số người dùng\")\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:23:53.55898Z","iopub.execute_input":"2025-07-26T08:23:53.559303Z","iopub.status.idle":"2025-07-26T08:23:53.842165Z","shell.execute_reply.started":"2025-07-26T08:23:53.559254Z","shell.execute_reply":"2025-07-26T08:23:53.841174Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Visualization cảm biến theo thời gian (Time Series Plot)","metadata":{}},{"cell_type":"code","source":"# Lấy ID người đầu tiên trong tập dữ liệu\npid = train_df['subject'].iloc[0]\n\n# Lấy toàn bộ dữ liệu của người đó\nsample = train_df[train_df['subject'] == pid]\n\n# Vẽ biểu đồ gia tốc kế theo thời gian\nplt.figure(figsize=(12, 5))\nfor col in ['acc_x', 'acc_y', 'acc_z']:\n    plt.plot(sample.index, sample[col], label=col)\n\nplt.title(f'Gia tốc kế của subject {pid}')\nplt.legend()\nplt.xlabel('Index (tương ứng thời gian)')\nplt.ylabel('Giá trị gia tốc (m/s²)')\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:26:57.262065Z","iopub.execute_input":"2025-07-26T08:26:57.262412Z","iopub.status.idle":"2025-07-26T08:26:57.553781Z","shell.execute_reply.started":"2025-07-26T08:26:57.262391Z","shell.execute_reply":"2025-07-26T08:26:57.552769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Phân tích cảm biến theo nhãn gesture","metadata":{}},{"cell_type":"code","source":"# Tạo cột trung bình cho mỗi ToF sensor\nfor i in range(1, 6):  # tof_1 đến tof_5\n    tof_v_cols = [col for col in train_df.columns if col.startswith(f'tof_{i}_v')]\n    train_df[f'tof_{i}_mean'] = train_df[tof_v_cols].replace(-1, np.nan).mean(axis=1)\n\n# Bổ sung các cột tof_mean vào cảm biến\ntof_mean_cols = [f'tof_{i}_mean' for i in range(1, 6)]\nsensor_cols = acc_cols + rot_cols + thm_cols + tof_mean_cols\n\n# Trung bình mỗi sensor theo gesture\ngesture_sensor_mean = train_df.groupby('gesture')[sensor_cols].mean()\n\n# Vẽ biểu đồ\ngesture_sensor_mean.T.plot(kind='bar', figsize=(16, 6))\nplt.title(\"Trung bình các cảm biến (gồm ToF) theo nhãn gesture\")\nplt.ylabel(\"Giá trị trung bình\")\nplt.xlabel(\"Cảm biến\")\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:27:27.08145Z","iopub.execute_input":"2025-07-26T08:27:27.081841Z","iopub.status.idle":"2025-07-26T08:27:30.130051Z","shell.execute_reply.started":"2025-07-26T08:27:27.081813Z","shell.execute_reply":"2025-07-26T08:27:30.128761Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Phân tích tương quan giữa các sensor","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Kiểm tra cột tồn tại và loại bỏ cột toàn NaN\nsensor_cols = acc_cols + rot_cols\nvalid_sensor_cols = [col for col in sensor_cols if col in train_df.columns and train_df[col].notna().any()]\n\n# Tính ma trận tương quan\ncorr_matrix = train_df[valid_sensor_cols].corr()\n\n# Vẽ heatmap\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, fmt=\".2f\", cmap='coolwarm', square=True, cbar_kws={'shrink': .8})\nplt.title(\"Ma trận tương quan giữa các cảm biến (acc + rot)\")\nplt.xticks(rotation=45)\nplt.yticks(rotation=0)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T08:28:45.762386Z","iopub.execute_input":"2025-07-26T08:28:45.762659Z","iopub.status.idle":"2025-07-26T08:28:46.133367Z","shell.execute_reply.started":"2025-07-26T08:28:45.762642Z","shell.execute_reply":"2025-07-26T08:28:46.132725Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"III. Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"Tính toán đặc trưng thống kê","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Ví dụ:\nsensor_cols = [c for c in train.columns if c.startswith(('acc_', 'rot_', 'thm_', 'tof_'))]\n\n# Group theo sequence và tính mean, std, min, max, median\nagg_funcs = ['mean', 'std', 'min', 'max', 'median']\n\nXtrain_stat = train.groupby('sequence_id')[sensor_cols].agg(agg_funcs)\nXtrain_stat.columns = ['_'.join(col) for col in Xtrain_stat.columns]  # flatten multi-index\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T09:24:22.620347Z","iopub.execute_input":"2025-07-26T09:24:22.620681Z","iopub.status.idle":"2025-07-26T09:24:30.587892Z","shell.execute_reply.started":"2025-07-26T09:24:22.620664Z","shell.execute_reply":"2025-07-26T09:24:30.587004Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" FFT (biến đổi Fourier)","metadata":{}},{"cell_type":"code","source":"from scipy.fft import fft\n\ndef compute_fft_features(df, signal_col):\n    fft_vals = np.abs(fft(df[signal_col]))\n    return {\n        f'{signal_col}_fft_mean': np.mean(fft_vals),\n        f'{signal_col}_fft_std': np.std(fft_vals),\n        f'{signal_col}_fft_max': np.max(fft_vals),\n        f'{signal_col}_fft_freq_max': np.argmax(fft_vals)\n    }\n\nfft_features = []\nfor seq_id, group in train.groupby('sequence_id'):\n    features = {}\n    for col in ['acc_x', 'acc_y', 'acc_z']:\n        features.update(compute_fft_features(group, col))\n    features['sequence_id'] = seq_id\n    fft_features.append(features)\n\nXtrain_fft = pd.DataFrame(fft_features).set_index('sequence_id')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T09:25:08.255602Z","iopub.execute_input":"2025-07-26T09:25:08.255974Z","iopub.status.idle":"2025-07-26T09:25:11.866529Z","shell.execute_reply.started":"2025-07-26T09:25:08.255948Z","shell.execute_reply":"2025-07-26T09:25:11.864606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Autocorrelation","metadata":{}},{"cell_type":"code","source":"def autocorr(x, lag=1):\n    return np.corrcoef(x[:-lag], x[lag:])[0,1] if len(x) > lag else 0\n\ndef compute_autocorr_features(df, signal_col):\n    return {\n        f'{signal_col}_autocorr1': autocorr(df[signal_col].values, lag=1),\n        f'{signal_col}_autocorr5': autocorr(df[signal_col].values, lag=5),\n    }\n\nauto_features = []\nfor seq_id, group in train.groupby('sequence_id'):\n    features = {}\n    for col in ['acc_x', 'acc_y', 'acc_z']:\n        features.update(compute_autocorr_features(group, col))\n    features['sequence_id'] = seq_id\n    auto_features.append(features)\n\nXtrain_auto = pd.DataFrame(auto_features).set_index('sequence_id')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T09:25:32.285209Z","iopub.execute_input":"2025-07-26T09:25:32.285618Z","iopub.status.idle":"2025-07-26T09:25:37.653594Z","shell.execute_reply.started":"2025-07-26T09:25:32.285597Z","shell.execute_reply":"2025-07-26T09:25:37.65222Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Peak detection (đỉnh chuyển động)","metadata":{}},{"cell_type":"code","source":"from scipy.signal import find_peaks\n\ndef count_peaks(signal, height=None, distance=10):\n    peaks, _ = find_peaks(signal, height=height, distance=distance)\n    return len(peaks)\n\npeak_features = []\nfor seq_id, group in train.groupby('sequence_id'):\n    features = {}\n    for col in ['acc_x', 'acc_y', 'acc_z']:\n        features[f'{col}_num_peaks'] = count_peaks(group[col].values)\n    features['sequence_id'] = seq_id\n    peak_features.append(features)\n\nXtrain_peaks = pd.DataFrame(peak_features).set_index('sequence_id')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T09:25:56.501156Z","iopub.execute_input":"2025-07-26T09:25:56.501522Z","iopub.status.idle":"2025-07-26T09:25:58.459827Z","shell.execute_reply.started":"2025-07-26T09:25:56.501499Z","shell.execute_reply":"2025-07-26T09:25:58.458256Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Sensor Fusion (Tổng hợp đa cảm biến)","metadata":{}},{"cell_type":"markdown","source":"Euclidean norm của acc và gyro","metadata":{}},{"cell_type":"code","source":"def compute_norm_features(df):\n    df = df.copy()\n    df['acc_norm'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n    df['rot_norm'] = np.sqrt(df['rot_x']**2 + df['rot_y']**2 + df['rot_z']**2)\n    return df[['acc_norm', 'rot_norm']]\n\nnorm_features = []\nfor seq_id, group in train.groupby('sequence_id'):\n    norm = compute_norm_features(group)\n    feats = {\n        'acc_norm_mean': norm['acc_norm'].mean(),\n        'acc_norm_std': norm['acc_norm'].std(),\n        'rot_norm_mean': norm['rot_norm'].mean(),\n        'rot_norm_std': norm['rot_norm'].std(),\n        'sequence_id': seq_id\n    }\n    norm_features.append(feats)\n\nXtrain_norm = pd.DataFrame(norm_features).set_index('sequence_id')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T09:26:31.185184Z","iopub.execute_input":"2025-07-26T09:26:31.185685Z","iopub.status.idle":"2025-07-26T09:26:46.389149Z","shell.execute_reply.started":"2025-07-26T09:26:31.185665Z","shell.execute_reply":"2025-07-26T09:26:46.38815Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"TOF & THM Distance/Spatial Features","metadata":{}},{"cell_type":"code","source":"from functools import reduce\n\nXtrain = reduce(lambda left, right: left.join(right, how='outer'), [\n    Xtrain_stat,\n    Xtrain_fft,\n    Xtrain_auto,\n    Xtrain_peaks,\n    Xtrain_norm\n])\n\n# Làm tương tự cho Xtest (sử dụng test.csv)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T09:27:24.468591Z","iopub.execute_input":"2025-07-26T09:27:24.468947Z","iopub.status.idle":"2025-07-26T09:27:24.618019Z","shell.execute_reply.started":"2025-07-26T09:27:24.468922Z","shell.execute_reply":"2025-07-26T09:27:24.616727Z"}},"outputs":[],"execution_count":null}]}