{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Overview","metadata":{}},{"cell_type":"markdown","source":"# 2. DATASET","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Loading the Datasets & Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport warnings\nfrom matplotlib.ticker import MaxNLocator\nfrom mpl_toolkits.mplot3d import Axes3D\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nfrom sklearn.metrics import accuracy_score, f1_score\nimport joblib\nfrom scipy.spatial.transform import Rotation as R","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:32.094572Z","iopub.execute_input":"2025-08-07T01:59:32.096141Z","iopub.status.idle":"2025-08-07T01:59:32.102434Z","shell.execute_reply.started":"2025-08-07T01:59:32.096061Z","shell.execute_reply":"2025-08-07T01:59:32.101583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the datasets\n#Dữ liệu cảm biến cho tập huấn luyện (Train Sensor Data): \ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\n#Thông tin nhân khẩu học của đối tượng trong tập huấn luyện (giới tính, tuổi, ...): \ntrain_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\n\n#Dữ liệu cảm biến cho tập kiểm tra:\ntest_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\")\n\n# Thông tin nhân khẩu học của đối tượng trong tập kiểm tra:\ntest_dem_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:32.104287Z","iopub.execute_input":"2025-08-07T01:59:32.104611Z","iopub.status.idle":"2025-08-07T01:59:56.108131Z","shell.execute_reply.started":"2025-08-07T01:59:32.104583Z","shell.execute_reply":"2025-08-07T01:59:56.107247Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.1 Khám phá dữ liệu","metadata":{}},{"cell_type":"markdown","source":"### 2.1.1 Mẫu dữ liệu \n","metadata":{}},{"cell_type":"code","source":"train_df.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:56.109022Z","iopub.execute_input":"2025-08-07T01:59:56.109288Z","iopub.status.idle":"2025-08-07T01:59:56.152672Z","shell.execute_reply.started":"2025-08-07T01:59:56.109266Z","shell.execute_reply":"2025-08-07T01:59:56.151815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dem_df.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:56.153583Z","iopub.execute_input":"2025-08-07T01:59:56.153937Z","iopub.status.idle":"2025-08-07T01:59:56.165063Z","shell.execute_reply.started":"2025-08-07T01:59:56.153914Z","shell.execute_reply":"2025-08-07T01:59:56.164220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:56.167446Z","iopub.execute_input":"2025-08-07T01:59:56.167759Z","iopub.status.idle":"2025-08-07T01:59:56.196529Z","shell.execute_reply.started":"2025-08-07T01:59:56.167736Z","shell.execute_reply":"2025-08-07T01:59:56.195508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dem_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:56.197595Z","iopub.execute_input":"2025-08-07T01:59:56.198040Z","iopub.status.idle":"2025-08-07T01:59:56.220379Z","shell.execute_reply.started":"2025-08-07T01:59:56.198011Z","shell.execute_reply":"2025-08-07T01:59:56.219282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T01:59:56.221777Z","iopub.execute_input":"2025-08-07T01:59:56.222116Z","iopub.status.idle":"2025-08-07T02:00:04.428915Z","shell.execute_reply.started":"2025-08-07T01:59:56.222065Z","shell.execute_reply":"2025-08-07T02:00:04.428161Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.2 Kích thước ","metadata":{}},{"cell_type":"code","source":"print(\"No. of rows : \", train_df.shape[0])\nprint(\"No. of columns : \", train_df.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:04.430019Z","iopub.execute_input":"2025-08-07T02:00:04.430343Z","iopub.status.idle":"2025-08-07T02:00:04.434705Z","shell.execute_reply.started":"2025-08-07T02:00:04.430310Z","shell.execute_reply":"2025-08-07T02:00:04.433735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"No. of rows : \", test_df.shape[0])\nprint(\"No. of columns : \", test_df.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:04.435495Z","iopub.execute_input":"2025-08-07T02:00:04.435775Z","iopub.status.idle":"2025-08-07T02:00:04.452618Z","shell.execute_reply.started":"2025-08-07T02:00:04.435754Z","shell.execute_reply":"2025-08-07T02:00:04.451735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"No. of rows : \", train_dem_df.shape[0])\nprint(\"No. of columns : \", train_dem_df.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:04.454038Z","iopub.execute_input":"2025-08-07T02:00:04.454341Z","iopub.status.idle":"2025-08-07T02:00:04.470268Z","shell.execute_reply.started":"2025-08-07T02:00:04.454322Z","shell.execute_reply":"2025-08-07T02:00:04.469408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"No. of rows : \", test_dem_df.shape[0])\nprint(\"No. of columns : \", test_dem_df.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:04.470994Z","iopub.execute_input":"2025-08-07T02:00:04.471247Z","iopub.status.idle":"2025-08-07T02:00:04.488452Z","shell.execute_reply.started":"2025-08-07T02:00:04.471228Z","shell.execute_reply":"2025-08-07T02:00:04.487527Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.3 Tóm tắt thống kê các tính năng ","metadata":{}},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:04.489342Z","iopub.execute_input":"2025-08-07T02:00:04.489763Z","iopub.status.idle":"2025-08-07T02:00:12.636153Z","shell.execute_reply.started":"2025-08-07T02:00:04.489741Z","shell.execute_reply":"2025-08-07T02:00:12.635308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dem_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:12.636938Z","iopub.execute_input":"2025-08-07T02:00:12.637180Z","iopub.status.idle":"2025-08-07T02:00:12.663377Z","shell.execute_reply.started":"2025-08-07T02:00:12.637162Z","shell.execute_reply":"2025-08-07T02:00:12.662591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:12.666682Z","iopub.execute_input":"2025-08-07T02:00:12.667287Z","iopub.status.idle":"2025-08-07T02:00:13.079835Z","shell.execute_reply.started":"2025-08-07T02:00:12.667264Z","shell.execute_reply":"2025-08-07T02:00:13.079055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dem_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:13.080739Z","iopub.execute_input":"2025-08-07T02:00:13.081038Z","iopub.status.idle":"2025-08-07T02:00:13.107000Z","shell.execute_reply.started":"2025-08-07T02:00:13.081017Z","shell.execute_reply":"2025-08-07T02:00:13.106274Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.4 Giá trị bị thiếu ","metadata":{}},{"cell_type":"code","source":"datasets = {\n \"Train Data\": train_df,\n \"Train Demographics\": train_dem_df,\n \"Test Data\": test_df,\n \"Test Demographics\": test_dem_df,\n}\n\nfor name, df in datasets.items():\n print(f\"Missing values in {name}:\")\n missing = df.isnull().sum()\n missing = missing[missing > 0]\n if not missing.empty:\n print(missing)\n else:\n print(\" ✅ No missing values.\")\n print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:13.107758Z","iopub.execute_input":"2025-08-07T02:00:13.108018Z","iopub.status.idle":"2025-08-07T02:00:14.029228Z","shell.execute_reply.started":"2025-08-07T02:00:13.107993Z","shell.execute_reply":"2025-08-07T02:00:14.028409Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.5 kiểm tra giá trị trùng lặp","metadata":{}},{"cell_type":"code","source":"#Kiểm tra trùng lặp (Duplicate Rows): .duplicated(): Trả về Series boolean, dòng nào trùng lặp sẽ là True.\n# .sum(): Đếm tổng số dòng trùng lặp.\n#Kiểm tra xem dữ liệu có bản ghi nào bị trùng không — quan trọng cho việc làm sạch dữ liệu.\n\n# Đếm các hàng trùng lặp trong train_df\ntrain_duplicates = train_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong test_df\ntest_duplicates = test_df.duplicated().sum()\n\n# Đếm các hàng trùng lặp trong train_dem_df (optional)\ntrain_dem_duplicates = train_dem_df.duplicated().sum()\n# Đếm các hàng trùng lặp trong test_dem_df (optional)\ntest_dem_duplicates = test_dem_df.duplicated().sum()\n\n# In số lượng dòng trùng:\nprint(f\"Number of duplicate rows in train_df: {train_duplicates}\")\nprint(f\"Number of duplicate rows in test_df: {test_duplicates}\")\nprint(f\"Number of duplicate rows in train_dem_df: {train_dem_duplicates}\")\nprint(f\"Number of duplicate rows in test_dem_df: {test_dem_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:14.030277Z","iopub.execute_input":"2025-08-07T02:00:14.030696Z","iopub.status.idle":"2025-08-07T02:00:19.468941Z","shell.execute_reply.started":"2025-08-07T02:00:14.030555Z","shell.execute_reply":"2025-08-07T02:00:19.468114Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.1.6 giá trị duy nhất","metadata":{}},{"cell_type":"code","source":"train_df.nunique().to_frame(\"# of unique values\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:19.470053Z","iopub.execute_input":"2025-08-07T02:00:19.470403Z","iopub.status.idle":"2025-08-07T02:00:22.349570Z","shell.execute_reply.started":"2025-08-07T02:00:19.470371Z","shell.execute_reply":"2025-08-07T02:00:22.348934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dem_df.nunique().to_frame(\"# of unique values\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:22.350388Z","iopub.execute_input":"2025-08-07T02:00:22.350618Z","iopub.status.idle":"2025-08-07T02:00:22.359535Z","shell.execute_reply.started":"2025-08-07T02:00:22.350600Z","shell.execute_reply":"2025-08-07T02:00:22.358690Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"markdown","source":"### Gesture (Target Column)","metadata":{}},{"cell_type":"code","source":"gesture_counts = train_df['gesture'].value_counts()\n\n# Calculate percentages\npercentages = (gesture_counts / gesture_counts.sum()) * 100\n\n# Create labels with both value counts and percentages\nlabels = [f'{label} ({count}, {perc:.1f}%)' for label, count, perc in zip(gesture_counts.index, gesture_counts.values, percentages)]\n\n# Create the pie chart\nplt.figure(figsize=(8, 8)) # Set figure size for better readability\nplt.pie(gesture_counts, labels=labels, startangle=160, counterclock=False, wedgeprops={'edgecolor': 'black'})\nplt.title('Distribution of Gestures')\nplt.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle.\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:22.360469Z","iopub.execute_input":"2025-08-07T02:00:22.360756Z","iopub.status.idle":"2025-08-07T02:00:22.981935Z","shell.execute_reply.started":"2025-08-07T02:00:22.360727Z","shell.execute_reply":"2025-08-07T02:00:22.980914Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Định hướng","metadata":{}},{"cell_type":"code","source":"# Donut Chart\nfig, ax = plt.subplots(figsize=(8, 8))\n\n# Calculate percentages for labels (optional, can also use autopct)\norientation_counts = train_df[\"orientation\"].value_counts().head()\ntotal_count = orientation_counts.sum()\nlabels = [f'{label} ({count})' for label, count in orientation_counts.items()]\n\nwedges, texts, autotexts = ax.pie(\n orientation_counts,\n autopct='%1.1f%%',\n startangle=90,\n colors=plt.cm.Paired.colors,\n pctdistance=0.85, # Distance of the percentage labels from the center\n wedgeprops=dict(width=0.3, edgecolor='w'), # Create the donut hole\n labels=labels\n)\n\n# Draw a circle in the middle to make it a donut chart\ncentre_circle = plt.Circle((0,0), 0.70, fc='white')\nfig.gca().add_artist(centre_circle)\n\nax.set_title(\"Subject's orientation (Donut Chart)\")\nax.axis('equal')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:22.982884Z","iopub.execute_input":"2025-08-07T02:00:22.983205Z","iopub.status.idle":"2025-08-07T02:00:23.185329Z","shell.execute_reply.started":"2025-08-07T02:00:22.983180Z","shell.execute_reply":"2025-08-07T02:00:23.184400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vertical Bar Chart\nfig, ax = plt.subplots(figsize=(8, 6))\norientation_counts.sort_values(ascending=False).plot( # Often good to sort descending for vertical bars\n kind=\"bar\", color='purple', ax=ax, title=\"Target (Vertical Bar)\"\n)\nax.set_xlabel(\"Target\")\nax.set_ylabel(\"Count\")\nplt.xticks(rotation=20, ha='right') # Rotate x-axis labels for better readability if they overlap\nplt.tight_layout() # Adjust layout to prevent labels from overlapping\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:23.186378Z","iopub.execute_input":"2025-08-07T02:00:23.186700Z","iopub.status.idle":"2025-08-07T02:00:23.452756Z","shell.execute_reply.started":"2025-08-07T02:00:23.186671Z","shell.execute_reply":"2025-08-07T02:00:23.451848Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sequence Type (Target vs Non Target)","metadata":{}},{"cell_type":"code","source":"# Donut Chart\nfig, ax = plt.subplots(figsize=(8, 8))\n\n# Calculate percentages for labels (optional, can also use autopct)\nsequence_type_counts = train_df[\"sequence_type\"].value_counts().head()\ntotal_count = sequence_type_counts.sum()\nlabels = [f'{label} ({count})' for label, count in sequence_type_counts.items()]\n\nwedges, texts, autotexts = ax.pie(\n sequence_type_counts,\n autopct='%1.1f%%',\n startangle=90,\n colors=plt.cm.Paired.colors,\n pctdistance=0.85, # Distance of the percentage labels from the center\n wedgeprops=dict(width=0.3, edgecolor='w'), # Create the donut hole\n labels=labels\n)\n\n# Draw a circle in the middle to make it a donut chart\ncentre_circle = plt.Circle((0,0), 0.70, fc='white')\nfig.gca().add_artist(centre_circle)\n\nax.set_title(\"Target vs Non-target (Donut Chart)\")\nax.axis('equal')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:23.453743Z","iopub.execute_input":"2025-08-07T02:00:23.454075Z","iopub.status.idle":"2025-08-07T02:00:23.636550Z","shell.execute_reply.started":"2025-08-07T02:00:23.454048Z","shell.execute_reply":"2025-08-07T02:00:23.635749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vertical Bar Chart\nfig, ax = plt.subplots(figsize=(8, 6))\nsequence_type_counts.sort_values(ascending=False).plot( # Often good to sort descending for vertical bars\n kind=\"bar\", color='lightgreen', ax=ax, title=\"Target vs Non Target (Vertical Bar)\"\n)\nax.set_xlabel(\"Target\")\nax.set_ylabel(\"Count\")\nplt.xticks(rotation=20, ha='right') # Rotate x-axis labels for better readability if they overlap\nplt.tight_layout() # Adjust layout to prevent labels from overlapping\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:23.637575Z","iopub.execute_input":"2025-08-07T02:00:23.637964Z","iopub.status.idle":"2025-08-07T02:00:23.820899Z","shell.execute_reply.started":"2025-08-07T02:00:23.637942Z","shell.execute_reply":"2025-08-07T02:00:23.820004Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Behavior","metadata":{}},{"cell_type":"code","source":"# Donut Chart\nfig, ax = plt.subplots(figsize=(8, 8))\n\n# Calculate percentages for labels (optional, can also use autopct)\nbehavior_counts = train_df[\"behavior\"].value_counts().head()\ntotal_count = behavior_counts.sum()\nlabels = [f'{label} ({count})' for label, count in behavior_counts.items()]\n\nwedges, texts, autotexts = ax.pie(\n behavior_counts,\n autopct='%1.1f%%',\n startangle=90,\n colors=plt.cm.Paired.colors,\n pctdistance=0.85, # Distance of the percentage labels from the center\n wedgeprops=dict(width=0.3, edgecolor='w'), # Create the donut hole\n labels=labels\n)\n\n# Draw a circle in the middle to make it a donut chart\ncentre_circle = plt.Circle((0,0), 0.70, fc='white')\nfig.gca().add_artist(centre_circle)\n\nax.set_title(\"Subject's Behavior During the Sequence (Donut Chart)\")\nax.axis('equal')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:23.821976Z","iopub.execute_input":"2025-08-07T02:00:23.822250Z","iopub.status.idle":"2025-08-07T02:00:24.035968Z","shell.execute_reply.started":"2025-08-07T02:00:23.822230Z","shell.execute_reply":"2025-08-07T02:00:24.035150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vertical Bar Chart\nfig, ax = plt.subplots(figsize=(8, 6))\nbehavior_counts.sort_values(ascending=False).plot( # Often good to sort descending for vertical bars\n kind=\"bar\", color='skyblue', ax=ax, title=\"Subject's Behavior During the Sequence (Vertical Bar)\"\n)\nax.set_xlabel(\"Behavior\")\nax.set_ylabel(\"Count\")\nplt.xticks(rotation=20, ha='right') # Rotate x-axis labels for better readability if they overlap\nplt.tight_layout() # Adjust layout to prevent labels from overlapping\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:24.036992Z","iopub.execute_input":"2025-08-07T02:00:24.037325Z","iopub.status.idle":"2025-08-07T02:00:24.228523Z","shell.execute_reply.started":"2025-08-07T02:00:24.037298Z","shell.execute_reply":"2025-08-07T02:00:24.227605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Demographics Correlation Matrix","metadata":{}},{"cell_type":"code","source":"# correlation matrix\nplt.figure(figsize = (8, 4))\nsns.heatmap(data = train_dem_df.corr(numeric_only = True), linewidth = 0.5, annot = True, linecolor = \"black\")\nplt.title('Train Data Demographic Correlation Heatmap')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:24.229585Z","iopub.execute_input":"2025-08-07T02:00:24.230047Z","iopub.status.idle":"2025-08-07T02:00:24.568140Z","shell.execute_reply.started":"2025-08-07T02:00:24.230020Z","shell.execute_reply":"2025-08-07T02:00:24.567237Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Gender Distribution","metadata":{}},{"cell_type":"code","source":"# Get value counts for gender\nvc = train_dem_df['sex'].value_counts()\n\n# Extract counts (assuming 'count' column doesn't exist)\ncounts = vc.values\n\n# Create labels (assuming 'count' column doesn't exist)\nlabels = ['Male', 'Female'] # Assuming 'Male' maps to '1' and 'Female' to '0'\n\n# Plot the pie chart\nplt.pie(counts, labels=labels, autopct=\"%1.1f%%\") # Add percentages to pie slices\nplt.title('Gender')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:24.569177Z","iopub.execute_input":"2025-08-07T02:00:24.569508Z","iopub.status.idle":"2025-08-07T02:00:24.653806Z","shell.execute_reply.started":"2025-08-07T02:00:24.569467Z","shell.execute_reply":"2025-08-07T02:00:24.653009Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Age Distribution with respect to Gender","metadata":{}},{"cell_type":"code","source":"#'train' is DataFrame\n_, axs = plt.subplots(2, 1, sharex=True)\n\nfor sex in range(2):\n ax = axs.ravel()[sex]\n\n # Filter by sex using boolean indexing\n sex_filter = train_dem_df['sex'] == sex\n vc = train_dem_df[sex_filter]['age'].value_counts() # Access column after filtering\n\n # Plot the bar chart\n ax.bar(vc.index, vc.values, color=['lightblue', 'coral'][sex], label=['Male', 'Female'][sex])\n ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n ax.set_ylabel('count')\n ax.legend()\n\nplt.suptitle('Age distribution with respect to Gender')\naxs.ravel()[1].set_xlabel('years')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:24.654723Z","iopub.execute_input":"2025-08-07T02:00:24.654952Z","iopub.status.idle":"2025-08-07T02:00:24.991295Z","shell.execute_reply.started":"2025-08-07T02:00:24.654934Z","shell.execute_reply":"2025-08-07T02:00:24.990322Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Linear Acceleration from IMU Sensor","metadata":{}},{"cell_type":"code","source":"# Initialize the 3D plot\nfig = plt.figure(figsize=(10, 10)) # Adjust figure size for better visibility\nax = fig.add_subplot(111, projection='3d')\n\n# Get unique values from the 'gesture' column for coloring\nunique_gestures = train_df['gesture'].unique()\n\n# Use a colormap to get distinct colors for each gesture type\n# matplotlib.colormaps is the recommended way to access colormaps now\ncolors = plt.colormaps['tab10'].resampled(len(unique_gestures)) # 'tab10' is good for categorical data\n\n# Iterate through unique gestures and plot them with different colors\nfor i, gesture_type in enumerate(unique_gestures):\n subset = train_df[train_df['gesture'] == gesture_type]\n ax.scatter(subset['acc_x'], subset['acc_y'], subset['acc_z'],\n color=colors(i), # Assign a unique color from the colormap\n label=gesture_type, # Label for the legend\n alpha=0.7, # Opacity (matches your Plotly opacity)\n s=2,\n rasterized=True) # Size of points (similar to Plotly's size_max=1, adjusted for visibility)\n\n# Set labels and title\nax.set_xlabel('acc_x', fontsize=14)\nax.set_ylabel('acc_y', fontsize=14)\nax.set_zlabel('acc_z', fontsize=14)\nax.set_title('Linear acceleration along X, Y, Z axes', fontsize=16)\n\n# Add a legend to differentiate 'gesture' types\nax.legend(title='Gesture Type', fontsize=10, title_fontsize=12, loc='upper right')\n\n# Set tick label font size\nax.tick_params(axis='both', which='major', labelsize=12)\n\n# --- Mimic Plotly's 'plotly_dark' template ---\n# Set the background color of the plot\nax.set_facecolor('black')\n\n# Make the pane background transparent\nax.xaxis.pane.fill = False\nax.yaxis.pane.fill = False\nax.zaxis.pane.fill = False\n\n# Set the pane edge colors\nax.xaxis.pane.edgecolor = 'grey'\nax.yaxis.pane.edgecolor = 'grey'\nax.zaxis.pane.edgecolor = 'grey'\n\n# Set the actual pane face colors to transparent\nax.xaxis.pane.set_facecolor((0.0, 0.0, 0.0, 0.0))\nax.yaxis.pane.set_facecolor((0.0, 0.0, 0.0, 0.0))\nax.zaxis.pane.set_facecolor((0.0, 0.0, 0.0, 0.0))\n\n# Set grid lines color\nax.grid(color='gray', linestyle='--', linewidth=0.5, alpha=0.5)\n\nplt.tight_layout() # Adjust layout to prevent labels from overlapping\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:24.992401Z","iopub.execute_input":"2025-08-07T02:00:24.992727Z","iopub.status.idle":"2025-08-07T02:00:35.235485Z","shell.execute_reply.started":"2025-08-07T02:00:24.992697Z","shell.execute_reply":"2025-08-07T02:00:35.234533Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3D Scatter plot shows linear accelartion along x, y and z axes. Color acts as the fourth dimension and differenciates different points based on the target column i.e gesture.","metadata":{}},{"cell_type":"code","source":"# Initialize the 3D plot\nfig = plt.figure(figsize=(10, 10)) # Adjust figure size for better visibility\nax = fig.add_subplot(111, projection='3d')\n\n# Get unique values from 'sequence_type' for coloring\nunique_sequence_types = train_df['sequence_type'].unique()\n\n# --- ADDRESSING DEPRECATION WARNING 1: get_cmap ---\n# Use matplotlib.colormaps[name] instead of plt.cm.get_cmap(name)\ncolors = plt.colormaps['viridis'].resampled(len(unique_sequence_types))\n\n# Iterate through unique sequence types and plot them with different colors\nfor i, seq_type in enumerate(unique_sequence_types):\n subset = train_df[train_df['sequence_type'] == seq_type]\n ax.scatter(subset['acc_x'], subset['acc_y'], subset['acc_z'],\n color=colors(i), # Assign a unique color\n label=seq_type, # Label for the legend\n alpha=0.7, # Opacity\n s=10) # Size of points (similar to Plotly's size_max)\n\n# Set labels and title\nax.set_xlabel('acc_x', fontsize=14)\nax.set_ylabel('acc_y', fontsize=14)\nax.set_zlabel('acc_z', fontsize=14)\nax.set_title('Linear acceleration along X, Y, Z axes', fontsize=16)\n\n# Add a legend to differentiate 'sequence_type'\nax.legend(title='Sequence Type', fontsize=12, title_fontsize=14)\n\n# Set tick label font size\nax.tick_params(axis='both', which='major', labelsize=12)\n\n# Set background color to dark and adjust grid/pane colors\nax.set_facecolor('black')\n\n# --- ADDRESSING DEPRECATION WARNINGS 2, 3, 4: w_xaxis, w_yaxis, w_zaxis ---\n# Use .xaxis, .yaxis, .zaxis directly for panes\nax.xaxis.pane.fill = False\nax.yaxis.pane.fill = False\nax.zaxis.pane.fill = False\n\nax.xaxis.pane.edgecolor = 'grey'\nax.yaxis.pane.edgecolor = 'grey'\nax.zaxis.pane.edgecolor = 'grey'\n\n# Set pane colors to transparent for a truly dark effect\n# These are now accessed directly through .xaxis.pane.set_facecolor\nax.xaxis.pane.set_facecolor((0.0, 0.0, 0.0, 0.0))\nax.yaxis.pane.set_facecolor((0.0, 0.0, 0.0, 0.0))\nax.zaxis.pane.set_facecolor((0.0, 0.0, 0.0, 0.0))\n\nax.grid(color='gray', linestyle='--', linewidth=0.5, alpha=0.5)\n\nplt.tight_layout() # Adjust layout to prevent labels from overlapping\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:35.236562Z","iopub.execute_input":"2025-08-07T02:00:35.236878Z","iopub.status.idle":"2025-08-07T02:00:48.615706Z","shell.execute_reply.started":"2025-08-07T02:00:35.236845Z","shell.execute_reply":"2025-08-07T02:00:48.614688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3D Scatter plot shows linear accelartion along x, y and z axes. Color acts as the fourth dimension and differenciates points based on sequence_type column i.e target and non-target.","metadata":{}},{"cell_type":"markdown","source":"### Rotation of Gyroscope from IMU Sensor","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(14, 8)) # Adjust figure size as needed\nax = fig.add_subplot(111, projection='3d')\n\n# Plot the 3D line\n# For color, matplotlib's plot3D doesn't directly support a continuous 'color'\n# argument like px.line_3d. You'd typically use a colormap and map 'rot_w' to it.\n# This example uses a simple line color. For continuous color, you'd need\n# to segment your data or use scatter3D with c=train['rot_w'].\n# ax.plot(train['rot_x'], train['rot_y'], train['rot_z'], color='skyblue', linewidth=0.5)\n\n# If you want to color by 'rot_w' with a gradient, it's often better to use scatter3D\n# if a continuous line isn't strictly necessary, or segment the line.\n# For a true line plot with continuous color, it's more complex in matplotlib.\n# Example for scatter3D with color:\nsc = ax.scatter(train_df['rot_x'], train_df['rot_y'], train_df['rot_z'], c=train_df['rot_w'], cmap='viridis', s=1)\nfig.colorbar(sc, ax=ax, label='rot_w')\n\n\nax.set_xlabel('rot_x')\nax.set_ylabel('rot_y')\nax.set_zlabel('rot_z')\nax.set_title('3D Path of Gyroscope Rotation Over Time')\n\n# Adjust font sizes if needed (similar to update_layout in Plotly)\nax.tick_params(axis='both', which='major', labelsize=12)\nax.xaxis.label.set_size(14)\nax.yaxis.label.set_size(14)\nax.zaxis.label.set_size(14)\nax.title.set_size(16)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:48.616869Z","iopub.execute_input":"2025-08-07T02:00:48.617276Z","iopub.status.idle":"2025-08-07T02:00:56.705395Z","shell.execute_reply.started":"2025-08-07T02:00:48.617232Z","shell.execute_reply":"2025-08-07T02:00:56.704564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Temperature from Thermopile of IMU Sensors","metadata":{}},{"cell_type":"code","source":"# Melt the DataFrame for easier plotting\ndf_melted = train_df.melt(id_vars='sequence_type', \n value_vars=['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5'],\n var_name='sensor', \n value_name='temperature')\n\n# Plot\nplt.figure(figsize=(10, 6))\nsns.boxplot(x='sequence_type', y='temperature', hue='sensor', data=df_melted)\nplt.title('Thermopile Sensor Readings by Sequence Type')\nplt.xlabel('Sequence Type')\nplt.ylabel('Temperature (°C)')\nplt.legend(title='Sensor')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:00:56.706476Z","iopub.execute_input":"2025-08-07T02:00:56.706736Z","iopub.status.idle":"2025-08-07T02:01:00.263809Z","shell.execute_reply.started":"2025-08-07T02:00:56.706716Z","shell.execute_reply":"2025-08-07T02:01:00.262751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by sequence_type and get mean values\nimport numpy as np\ndf_grouped = train_df.groupby('sequence_type')[['thm_1','thm_2','thm_3','thm_4','thm_5']].mean()\n\ncategories = ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']\nN = len(categories)\n\nangles = [n / float(N) * 2 * np.pi for n in range(N)]\nangles += angles[:1] # Close the loop\n\nfig, ax = plt.subplots(figsize=(6, 6), subplot_kw=dict(polar=True))\n\nfor idx, row in df_grouped.iterrows():\n values = row.values.tolist()\n values += values[:1] # Close the loop\n ax.plot(angles, values, marker='o', label=idx)\n ax.fill(angles, values, alpha=0.1)\n\nax.set_xticks(angles[:-1])\nax.set_xticklabels(categories)\nplt.legend(title=\"Sequence Type\")\nplt.title(\"Radar Chart of Average Thermopile Readings by Sequence Type\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:00.264721Z","iopub.execute_input":"2025-08-07T02:01:00.264995Z","iopub.status.idle":"2025-08-07T02:01:00.801858Z","shell.execute_reply.started":"2025-08-07T02:01:00.264973Z","shell.execute_reply":"2025-08-07T02:01:00.800905Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"By analyzing the Boxplot and Radar Chart above we can say that temperature for target is slightly higher than non-target. Next lets see box plot of thermopile values against column gesture and column behavior.","metadata":{}},{"cell_type":"code","source":"# Melt the DataFrame for easier plotting\ndf_melted = train_df.melt(id_vars='gesture', \n value_vars=['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5'],\n var_name='sensor', \n value_name='temperature')\n\n# Plot\nplt.figure(figsize=(10, 6))\nsns.boxplot(x='gesture', y='temperature', hue='sensor', data=df_melted)\nplt.title('Thermopile Sensor Readings by Gesture')\nplt.xlabel('Gesture')\nplt.ylabel('Temperature (°C)')\nplt.legend(title='Sensor')\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:00.802850Z","iopub.execute_input":"2025-08-07T02:01:00.803564Z","iopub.status.idle":"2025-08-07T02:01:06.878367Z","shell.execute_reply.started":"2025-08-07T02:01:00.803500Z","shell.execute_reply":"2025-08-07T02:01:06.877420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Melt the DataFrame for easier plotting\ndf_melted = train_df.melt(id_vars='behavior', \n value_vars=['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5'],\n var_name='sensor', \n value_name='temperature')\n\n# Plot\nplt.figure(figsize=(10, 6))\nsns.boxplot(x='behavior', y='temperature', hue='sensor', data=df_melted)\nplt.title('Thermopile Sensor Readings by Behavior')\nplt.xlabel('Behavior')\nplt.ylabel('Temperature (°C)')\nplt.legend(title='Sensor')\nplt.xticks(rotation=25)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:06.879276Z","iopub.execute_input":"2025-08-07T02:01:06.879528Z","iopub.status.idle":"2025-08-07T02:01:10.861840Z","shell.execute_reply.started":"2025-08-07T02:01:06.879509Z","shell.execute_reply":"2025-08-07T02:01:10.860918Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Null Percentage","metadata":{}},{"cell_type":"code","source":"def null_percent(df):\n per=((df.isnull().sum()/len(df))*100).round(2)\n return per[per>0]\n\nprint(\"Nan Values in Train data\")\nprint(null_percent(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:10.862752Z","iopub.execute_input":"2025-08-07T02:01:10.863129Z","iopub.status.idle":"2025-08-07T02:01:11.746301Z","shell.execute_reply.started":"2025-08-07T02:01:10.863080Z","shell.execute_reply":"2025-08-07T02:01:11.745515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Initial Observations or Trends","metadata":{}},{"cell_type":"code","source":"#Tổng hợp dữ liệu vào dictionary: Tạo dictionary datasets chứa 4 bộ dữ liệu chính, giúp dễ quản lý và xử lý lặp đi lặp lại.\ndatasets = {\n \"Train Data\": train_df,\n \"Train Demographics\": train_dem_df,\n \"Test Data\": test_df,\n \"Test Demographics\": test_dem_df,\n}\n\n# Print shapes\n#Giải thích: .items(): Duyệt qua từng cặp tên và DataFrame.\n# .shape: Trả về tuple (số dòng, số cột).\n# print(...): In ra tên dataset, số dòng và số cột, giúp bạn hình dung dữ liệu có bao nhiêu bản ghi và thuộc tính.\n\n\nfor name, df in datasets.items():\n num_rows, num_cols = df.shape\n print(f\"{name}:\")\n print(f\" Number of Rows: {num_rows}\")\n print(f\" Number of Columns: {num_cols}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:11.747232Z","iopub.execute_input":"2025-08-07T02:01:11.747539Z","iopub.status.idle":"2025-08-07T02:01:11.754006Z","shell.execute_reply.started":"2025-08-07T02:01:11.747509Z","shell.execute_reply":"2025-08-07T02:01:11.753142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.4 Hợp nhất dữ liệu nhân khẩu học và cảm biến ---\nmerged_train = pd.merge(train_df, train_dem_df, on=\"subject\", how=\"left\")\nmerged_test = pd.merge(test_df, test_dem_df, on=\"subject\", how=\"left\")\n\nprint(\"Merged Train Shape:\", merged_train.shape)\nprint(\"Merged Test Shape:\", merged_test.shape)\ndisplay(merged_train.head(2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:11.754844Z","iopub.execute_input":"2025-08-07T02:01:11.755158Z","iopub.status.idle":"2025-08-07T02:01:12.439327Z","shell.execute_reply.started":"2025-08-07T02:01:11.755131Z","shell.execute_reply":"2025-08-07T02:01:12.438296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Thống kê mô tả toàn bộ merged_train ---\nmerged_train.describe().T\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:12.440528Z","iopub.execute_input":"2025-08-07T02:01:12.440958Z","iopub.status.idle":"2025-08-07T02:01:22.004422Z","shell.execute_reply.started":"2025-08-07T02:01:12.440929Z","shell.execute_reply":"2025-08-07T02:01:22.003648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.5 Phân phối hành vi ---\nsns.countplot(data=merged_train, x='behavior', order=merged_train['behavior'].value_counts().index)\nplt.title(\"Phân phối các hành vi trong tập huấn luyện\")\nplt.xticks(rotation=45)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:22.010253Z","iopub.execute_input":"2025-08-07T02:01:22.011021Z","iopub.status.idle":"2025-08-07T02:01:22.632120Z","shell.execute_reply.started":"2025-08-07T02:01:22.010995Z","shell.execute_reply":"2025-08-07T02:01:22.631269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.12 Behavior theo phase & sequence_type ---\nsns.countplot(data=merged_train, x='behavior', hue='phase')\nplt.title(\"Behavior theo Phase\")\nplt.xticks(rotation=45)\nplt.show()\n\nsns.countplot(data=merged_train, x='behavior', hue='sequence_type')\nplt.title(\"Behavior theo Sequence Type\")\nplt.xticks(rotation=45)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:22.632989Z","iopub.execute_input":"2025-08-07T02:01:22.633279Z","iopub.status.idle":"2025-08-07T02:01:24.170297Z","shell.execute_reply.started":"2025-08-07T02:01:22.633259Z","shell.execute_reply":"2025-08-07T02:01:24.169395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.10 Phát hiện Outlier trong độ tuổi ---\nplt.figure(figsize=(6, 3))\nsns.boxplot(x=merged_train['age'])\nplt.title(\"Boxplot - Độ tuổi\")\nplt.show()\n\n# Lọc nếu cần\nmerged_train = merged_train[merged_train['age'] <= 100]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:24.171129Z","iopub.execute_input":"2025-08-07T02:01:24.171368Z","iopub.status.idle":"2025-08-07T02:01:25.939905Z","shell.execute_reply.started":"2025-08-07T02:01:24.171349Z","shell.execute_reply":"2025-08-07T02:01:25.939246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.6 Biểu đồ histogram các đặc trưng cảm biến acc_ ---\nacc_cols = [col for col in merged_train.columns if col.startswith('acc_')]\nmerged_train[acc_cols].hist(figsize=(16, 12), bins=30)\nplt.suptitle(\"Phân phối các đặc trưng gia tốc (acc_)\", fontsize=16)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:25.940821Z","iopub.execute_input":"2025-08-07T02:01:25.941138Z","iopub.status.idle":"2025-08-07T02:01:26.716975Z","shell.execute_reply.started":"2025-08-07T02:01:25.941111Z","shell.execute_reply":"2025-08-07T02:01:26.716012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.7 Ma trận tương quan acc_ ---\nplt.figure(figsize=(12, 10))\nsns.heatmap(merged_train[acc_cols].corr(), cmap='coolwarm', center=0)\nplt.title(\"Tương quan giữa các đặc trưng gia tốc (acc_)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:26.717933Z","iopub.execute_input":"2025-08-07T02:01:26.718244Z","iopub.status.idle":"2025-08-07T02:01:26.960388Z","shell.execute_reply.started":"2025-08-07T02:01:26.718223Z","shell.execute_reply":"2025-08-07T02:01:26.959548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2.8.1 Phân phối hành vi theo giới tính ---\nsns.countplot(data=merged_train, x='behavior', hue='sex', order=merged_train['behavior'].value_counts().index)\nplt.title(\"Phân phối hành vi theo giới tính\")\nplt.xticks(rotation=45)\nplt.show()\n\n# --- 2.8.2 Phân phối tuổi theo hành vi ---\nsns.boxplot(data=merged_train, x='behavior', y='age')\nplt.title(\"Tuổi trung bình theo hành vi\")\nplt.xticks(rotation=45)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:26.961230Z","iopub.execute_input":"2025-08-07T02:01:26.961441Z","iopub.status.idle":"2025-08-07T02:01:27.950778Z","shell.execute_reply.started":"2025-08-07T02:01:26.961424Z","shell.execute_reply":"2025-08-07T02:01:27.949848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Hiển thị dữ liệu mẫu: .head(2): Xem trước 2 dòng đầu của mỗi bảng.\n# display(...): Dùng trong notebook để hiển thị bảng dữ liệu đẹp, rõ ràng.\n\ndisplay(train_df.head(2))\ndisplay(train_dem_df.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:27.951817Z","iopub.execute_input":"2025-08-07T02:01:27.952084Z","iopub.status.idle":"2025-08-07T02:01:27.976985Z","shell.execute_reply.started":"2025-08-07T02:01:27.952063Z","shell.execute_reply":"2025-08-07T02:01:27.976153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(test_df.head(2))\ndisplay(test_dem_df.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:27.978012Z","iopub.execute_input":"2025-08-07T02:01:27.978374Z","iopub.status.idle":"2025-08-07T02:01:28.011069Z","shell.execute_reply.started":"2025-08-07T02:01:27.978342Z","shell.execute_reply":"2025-08-07T02:01:28.010288Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.2.1. Participant & Contextual Features (Tính năng của người tham gia và bối cảnh)","metadata":{}},{"cell_type":"code","source":"#Chuẩn bị hàm thống kê mô tả có lọc cột: \n# excluded_prefixes: Bộ tiền tố để loại các cột cảm biến như: 'acc_': Gia tốc kế, 'rot_': Cảm biến xoay, 'thm_': Nhiệt độ/Độ ẩm. 'tof_': Time-of-Flight (cảm biến khoảng cách).\n\n# Define excluded prefixes\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\n\n#Hàm filtered_describe: Lọc các cột không thuộc cảm biến và có kiểu số.\n# Dùng .describe().T để thống kê mô tả (mean, std, min, max,...) và chuyển vị bảng cho dễ nhìn.\n# display: Hiển thị bảng thống kê.\n\ndef filtered_describe(df, name):\n # Exclude sensor columns\n filtered_cols = [col for col in df.columns \n if not col.startswith(excluded_prefixes) and pd.api.types.is_numeric_dtype(df[col])]\n \n # Describe and style\n print(f'\\n➡️ Description of numerical columns in {name}')\n return df[filtered_cols].describe().T.style.background_gradient(cmap='viridis')\n\n# Only analyze train sets!\ndisplay(filtered_describe(train_df, \"train_df\"))\ndisplay(filtered_describe(train_dem_df, \"train_dem_df\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:28.012319Z","iopub.execute_input":"2025-08-07T02:01:28.012625Z","iopub.status.idle":"2025-08-07T02:01:28.162808Z","shell.execute_reply.started":"2025-08-07T02:01:28.012593Z","shell.execute_reply":"2025-08-07T02:01:28.161889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kiểm tra thiếu giá trị & thống kê cảm biến: \n#Danh sách các cột không thuộc cảm biến (có thể là ID, thông tin khác). \n\nexcluded_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nsensor_cols = [col for col in train_df.columns if not col.startswith(excluded_prefixes)]\n\n# Sensor Data Summary for TRAIN\n#isnull().sum(): Đếm số giá trị bị thiếu.\nmissing_sensor_train = pd.DataFrame({\n 'Feature': sensor_cols,\n '[TRAIN] Missing Count': train_df[sensor_cols].isnull().sum().values,\n '[TRAIN] Missing %': (train_df[sensor_cols].isnull().sum().values / len(train_df)) * 100\n})\n\n#nunique(): Đếm số lượng giá trị duy nhất.\nunique_sensor_train = pd.DataFrame({\n 'Feature': sensor_cols,\n 'Unique Values [TRAIN]': train_df[sensor_cols].nunique().values\n})\n\n#dtypes: Lấy kiểu dữ liệu của từng cột.\ndtypes_sensor = pd.DataFrame({\n 'Feature': sensor_cols,\n 'Data Type': train_df[sensor_cols].dtypes.values\n})\n\n# Merge all summaries (NO test set)\n#merge: Gộp các bảng thống kê thành bảng duy nhất theo Feature.\nsensor_summary = missing_sensor_train \\\n .merge(unique_sensor_train, on='Feature', how='left') \\\n .merge(dtypes_sensor, on='Feature', how='left')\n\n# Display styled DataFrame (mask NaNs just for styling)\n#fillna(0): Điền giá trị thiếu bằng 0 (cho đẹp mắt khi hiển thị).\n#.style.background_gradient: Tô màu nền theo giá trị giúp dễ nhìn.\nstyled_df = sensor_summary.fillna(0)\nstyled_df.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:28.163674Z","iopub.execute_input":"2025-08-07T02:01:28.164038Z","iopub.status.idle":"2025-08-07T02:01:29.202971Z","shell.execute_reply.started":"2025-08-07T02:01:28.164018Z","shell.execute_reply":"2025-08-07T02:01:29.202290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Thống kê tương tự cho nhân khẩu học: Tương tự như phần thống kê cảm biến nhưng áp dụng cho dữ liệu nhân khẩu học.\n\n# Cột nhân khẩu học (không loại trừ)\ndem_cols = train_dem_df.columns\n\n# Giá trị bị thiếu trong dữ liệu nhân khẩu học của train \nmissing_demo_train = pd.DataFrame({\n 'Feature': dem_cols,\n '[TRAIN DEMO] Missing Count': train_dem_df[dem_cols].isnull().sum().values,\n '[TRAIN DEMO] Missing %': (train_dem_df[dem_cols].isnull().sum().values / len(train_dem_df)) * 100\n})\n\n# Giá trị duy nhất được tính trong dữ liệu nhân khẩu học của train \nunique_demo_train = pd.DataFrame({\n 'Feature': dem_cols,\n 'Unique Values [TRAIN DEMO]': train_dem_df[dem_cols].nunique().values\n})\n\n# Data types\ndtypes_demo = pd.DataFrame({\n 'Feature': dem_cols,\n 'Data Type': train_dem_df[dem_cols].dtypes.values\n})\n\n# Merge summaries (train only)\ndemo_summary = (\n missing_demo_train\n .merge(unique_demo_train, on='Feature', how='left')\n .merge(dtypes_demo, on='Feature', how='left')\n)\n\n# Display styled summary\ndemo_summary.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:29.203938Z","iopub.execute_input":"2025-08-07T02:01:29.204237Z","iopub.status.idle":"2025-08-07T02:01:29.230021Z","shell.execute_reply.started":"2025-08-07T02:01:29.204208Z","shell.execute_reply":"2025-08-07T02:01:29.229166Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2.2.2. Sensor-Based Features (Tính năng dựa trên cảm biến)","metadata":{}},{"cell_type":"markdown","source":"#### **NOTE:**\n* acc_: Gia tốc kế.\n* rot_: Cảm biến quay.\n* thm_: Nhiệt độ, độ ẩm.\n* tof_: Khoảng cách.\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Sensor column groups (use train_df only)\n# Phân nhóm các cột cảm biến:\nacc_cols = [col for col in train_df.columns if col.startswith('acc_')]\nrot_cols = [col for col in train_df.columns if col.startswith('rot_')]\nthm_cols = [col for col in train_df.columns if col.startswith('thm_')]\ntof_cols = [col for col in train_df.columns if col.startswith('tof_')]\n\n# Helper function for summary stats (train only)\n#Hàm thống kê chi tiết từng nhóm cảm biến (Tỷ lệ thiếu giá trị; Giá trị nhỏ nhất, lớn nhất; Trung bình, độ lệch chuẩn.)\ndef sensor_summary(df, cols, name, dataset_name):\n summary = pd.DataFrame({\n 'Feature': cols,\n f'{dataset_name} Missing %': df[cols].isnull().mean().values * 100,\n f'{dataset_name} Min': df[cols].min().values,\n f'{dataset_name} Max': df[cols].max().values,\n f'{dataset_name} Mean': df[cols].mean().values,\n f'{dataset_name} Std': df[cols].std().values\n })\n summary.insert(0, 'Sensor', name)\n return summary\n\n# Compute train summary for each sensor type\n#Thống kê tổng hợp cho tập train: Duyệt từng nhóm cảm biến và gom kết quả thống kê lại.\n# pd.concat: Ghép bảng lại thành bảng thống kê tổng hợp.\ndef train_sensor_summary(train_df, sensor_cols_dict):\n all_train = []\n for sensor_name, cols in sensor_cols_dict.items():\n all_train.append(sensor_summary(train_df, cols, sensor_name, 'Train'))\n train_summary = pd.concat(all_train, ignore_index=True)\n return train_summary\n\n# Sensor groups to process\nsensor_cols_dict = {\n 'acc': acc_cols,\n 'rot': rot_cols,\n 'thm': thm_cols,\n 'tof': tof_cols\n}\n\n# Run train summary only\nsensor_train_summary = train_sensor_summary(train_df, sensor_cols_dict)\n\n#rung bình theo nhóm cảm biến: \n# .groupby('Sensor'): Gom nhóm theo loại cảm biến.\n# .mean(numeric_only=True): Trung bình các chỉ số số học.\n# Tô màu bảng kết quả giúp dễ phân tích.\n\n# Show sensor-level averages for train\nsummary_by_group = sensor_train_summary.groupby('Sensor').mean(numeric_only=True)\n\n# Display styled table\nsummary_by_group.style.background_gradient(cmap='viridis')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:29.231067Z","iopub.execute_input":"2025-08-07T02:01:29.231665Z","iopub.status.idle":"2025-08-07T02:01:34.987945Z","shell.execute_reply.started":"2025-08-07T02:01:29.231635Z","shell.execute_reply":"2025-08-07T02:01:34.987232Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. EDA","metadata":{}},{"cell_type":"markdown","source":"## 3.1. Numerical Feature Distribution Analysis (Phân tích phân phối tính năng số) (Phân tích đơn biến)","metadata":{}},{"cell_type":"markdown","source":"### 3.1.1 Participant & Contextual Features (Tính năng của người tham gia và bối cảnh)","metadata":{}},{"cell_type":"code","source":"#Ẩn cảnh báo không cần thiết: Ẩn cảnh báo tương lai của thư viện, Ẩn cảnh báo về ký tự bị thiếu font khi vẽ biểu đồ.\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", message=\"Glyph.*missing from current font\")\n\ncustom_palette = ['#3498db', '#e74c3c', '#2ecc71']\nsensor_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\ncategorical_columns = ['phase', 'behavior', 'orientation', 'sequence_type',\n 'adult_child', 'sex', 'handedness']\ntarget_columns = ['gesture']\ncategorical_tracker = categorical_columns.copy()\n\ndef get_numerical_variables(df, excluded_prefixes, excluded_columns):\n return [col for col in df.columns \n if pd.api.types.is_numeric_dtype(df[col])\n and not col.startswith(excluded_prefixes)\n and col not in excluded_columns]\n\n# ONLY TRAIN SETS!\ntrain_main = train_df.copy()\ntrain_demo = train_dem_df.copy()\n\ntrain_main['Dataset'] = 'Train'\ntrain_demo['Dataset'] = 'Train'\n\nmain_data = train_main\ndemo_data = train_demo\n\nmain_numeric_vars = get_numerical_variables(main_data, sensor_prefixes, categorical_columns)\ndemo_numeric_vars = get_numerical_variables(demo_data, (), categorical_columns)\n\ndef create_variable_plots(variable, dataset_label, data):\n sns.set_style('whitegrid')\n fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n\n # Box plot\n plt.subplot(1, 2, 1)\n sns.boxplot(data=data, x=variable, y=\"Dataset\", palette=custom_palette)\n plt.xlabel(variable)\n plt.title(f\" Box Plot for {variable} — {dataset_label}\")\n\n # Histogram\n plt.subplot(1, 2, 2)\n for label, color in zip(data['Dataset'].unique(), custom_palette):\n subset = data[data['Dataset'] == label]\n sns.histplot(data=subset, x=variable, kde=True, bins=30, label=label, color=color)\n plt.xlabel(variable)\n plt.ylabel(\"Frequency\")\n plt.title(f\" Histogram for {variable} — {dataset_label}\")\n plt.legend()\n\n plt.tight_layout()\n plt.show()\n\n# Plot only train set context features\nfor var in main_numeric_vars:\n create_variable_plots(var, \"Context Features\", main_data)\n\n# Plot only train set demographic features\nfor var in demo_numeric_vars:\n create_variable_plots(var, \"Demographic Features\", demo_data)\n\n# --- Cleanup\ntrain_main.drop('Dataset', axis=1, inplace=True)\ntrain_demo.drop('Dataset', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:34.988903Z","iopub.execute_input":"2025-08-07T02:01:34.989370Z","iopub.status.idle":"2025-08-07T02:01:42.504633Z","shell.execute_reply.started":"2025-08-07T02:01:34.989342Z","shell.execute_reply":"2025-08-07T02:01:42.503706Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.1.2 Sensor-Based Features (Tính năng dựa trên cảm biến)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# 1) Copy train and test so we don’t modify original DataFrames\ntrain_temp = train_df.copy()\ntest_temp = test_df.copy()\n\n# 2) ACCELEROMETER: compute magnitude at each timestamp\ntrain_temp['acc_mag'] = np.sqrt(\n train_temp['acc_x']**2 + train_temp['acc_y']**2 + train_temp['acc_z']**2\n)\ntest_temp['acc_mag'] = np.sqrt(\n test_temp['acc_x']**2 + test_temp['acc_y']**2 + test_temp['acc_z']**2\n)\n\n# 3) ROTATION: compute “rotation angle” from quaternion w component\n# (Note: rot_w is in [-1,1], so arccos is valid. We ignore NaNs if any.)\ntrain_temp['rot_angle'] = 2 * np.arccos(train_temp['rot_w'].clip(-1,1))\ntest_temp['rot_angle'] = 2 * np.arccos(test_temp['rot_w'].clip(-1,1))\n\n# 4) Group by sequence_id and aggregate accelerometer summaries\nacc_agg_funcs = {\n 'acc_mag': ['mean', 'std', 'max']\n}\ntrain_acc_summary = train_temp.groupby('sequence_id').agg(acc_agg_funcs)\ntest_acc_summary = test_temp.groupby('sequence_id').agg(acc_agg_funcs)\n\n# Flatten column MultiIndex\ntrain_acc_summary.columns = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\ntest_acc_summary.columns = ['acc_mag_' + stat for stat in ['mean', 'std', 'max']]\n\n# 5) Group by sequence_id and aggregate rotation summaries\nrot_agg_funcs = {\n 'rot_angle': ['mean', 'std', 'max']\n}\ntrain_rot_summary = train_temp.groupby('sequence_id').agg(rot_agg_funcs)\ntest_rot_summary = test_temp.groupby('sequence_id').agg(rot_agg_funcs)\n\ntrain_rot_summary.columns = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\ntest_rot_summary.columns = ['rot_angle_' + stat for stat in ['mean', 'std', 'max']]\n\n# 6) THERMOPILE: five sensors thm_1 … thm_5\nthm_cols = [f\"thm_{i}\" for i in range(1, 6)]\n\n# Define aggregation functions: mean + std\nthm_agg_funcs = {col: ['mean', 'std'] for col in thm_cols}\n\ntrain_thm_summary = train_temp.groupby('sequence_id').agg(thm_agg_funcs)\ntest_thm_summary = test_temp.groupby('sequence_id').agg(thm_agg_funcs)\n\n# Flatten MultiIndex columns\nflattened_thm_cols = []\nfor sensor in thm_cols:\n for stat in ['mean','std']:\n flattened_thm_cols.append(f\"{sensor}_{stat}\")\n\ntrain_thm_summary.columns = flattened_thm_cols\ntest_thm_summary.columns = flattened_thm_cols\n\n# 7) TIME‐OF‐FLIGHT: each sensor i has 64 pixel columns: tof_i_v0 … tof_i_v63\n# We'll create one “tof_i_mean_at_ts” per timestamp, then aggregate per sequence.\n\ndef compute_tof_sequence_summary(df):\n # Initialize a dict to hold the per‐sequence DataFrames\n seq_summaries = {}\n\n for i in range(1, 6):\n # Build a list of columns for sensor i\n tof_cols = [f\"tof_{i}_v{pix}\" for pix in range(64)]\n # Replace -1 with NaN so they don't skew the mean; cast to float\n ts_grid = df[tof_cols].replace(-1, np.nan).astype(float)\n # Compute “mean across all 64 pixels” for each timestamp\n df[f\"tof_{i}_mean_at_ts\"] = ts_grid.mean(axis=1)\n \n # Now, group by sequence_id and compute per‐sequence mean & std of those means\n agg_dict = {f\"tof_{i}_mean_at_ts\": ['mean','std'] for i in range(1, 6)}\n summary = df.groupby('sequence_id').agg(agg_dict)\n # Flatten MultiIndex columns\n flat_cols = [f\"tof_{i}_{stat}\" for i in range(1, 6) for stat in ['mean','std']]\n summary.columns = flat_cols\n return summary\n\ntrain_tof_summary = compute_tof_sequence_summary(train_temp)\ntest_tof_summary = compute_tof_sequence_summary(test_temp)\n\n# 8) Merge accel, rotation, thm, tof summaries (on sequence_id)\ntrain_sensor_summary = (\n train_acc_summary\n .join(train_rot_summary, how='outer')\n .join(train_thm_summary, how='outer')\n .join(train_tof_summary, how='outer')\n)\n\ntest_sensor_summary = (\n test_acc_summary\n .join(test_rot_summary, how='outer')\n .join(test_thm_summary, how='outer')\n .join(test_tof_summary, how='outer')\n)\n\n# 9) Add “Dataset” column so we can do box+hist side‐by‐side\ntrain_sensor_summary['Dataset'] = 'Train'\ntest_sensor_summary['Dataset'] = 'Test'\n\n# 10) Concatenate into one DataFrame for plotting\ncombined_sensor_summary = pd.concat(\n [train_sensor_summary, test_sensor_summary],\n axis=0\n).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:42.505771Z","iopub.execute_input":"2025-08-07T02:01:42.506062Z","iopub.status.idle":"2025-08-07T02:01:48.444962Z","shell.execute_reply.started":"2025-08-07T02:01:42.506040Z","shell.execute_reply":"2025-08-07T02:01:48.444004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Re-use your custom palette\ncustom_palette = ['#3498db', '#e74c3c', '#2ecc71']\n\n# Filter only \"Train\" samples\ntrain_sensor_summary = combined_sensor_summary[combined_sensor_summary['Dataset'] == 'Train']\n\n# Define the plotting function (now uses only train_sensor_summary)\ndef create_sensor_summary_plots(variable, dataset_label, data):\n sns.set_style('whitegrid')\n fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n\n # Box Plot (left)\n plt.subplot(1, 2, 1)\n sns.boxplot(data=data, x=variable, y=\"Dataset\", palette=[custom_palette[0]])\n plt.xlabel(variable)\n plt.title(f\"Box Plot for {variable} — {dataset_label}\")\n\n # Histogram (right)\n plt.subplot(1, 2, 2)\n sns.histplot(data=data, x=variable, kde=True, bins=30, color=custom_palette[0], label=\"Train\")\n plt.xlabel(variable)\n plt.ylabel(\"Frequency\")\n plt.title(f\"Histogram for {variable} — {dataset_label}\")\n plt.legend()\n\n plt.tight_layout()\n plt.show()\n\n# List all sensor-summary variables (exclude 'Dataset')\nsensor_summary_vars = [col for col in train_sensor_summary.columns if col != 'Dataset']\n\n# Plot each\nfor var in sensor_summary_vars:\n create_sensor_summary_plots(var, \"Per-Sequence Sensor Summaries\", train_sensor_summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:01:48.446031Z","iopub.execute_input":"2025-08-07T02:01:48.446363Z","iopub.status.idle":"2025-08-07T02:02:04.861377Z","shell.execute_reply.started":"2025-08-07T02:01:48.446332Z","shell.execute_reply":"2025-08-07T02:02:04.860391Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Categorical Feature Analysis (Univariate Analysis)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport textwrap\n\n# Define palettes\npie_chart_palette = ['#33638d', '#28ae80', '#d3eb0c', '#ff9a0b', '#7e03a8', '#35b779',\n '#fde725', '#440154', '#90d743', '#482173', '#22a884', '#f8961e']\ncustom_palette = ['#3498db'] # Only Train\n\n# 🔹 Categorical variables to analyze (excluding 'gesture')\ncategorical_variables = [col for col in categorical_tracker if col != 'gesture']\n\n# --- Prep main dataset (train only)\ntrain_main = train_df.copy()\ntrain_main['dataset'] = 'train'\nmain_combined = train_main.copy()\n\n# --- Prep demographic dataset (train only)\ntrain_demo = train_dem_df.copy()\ntrain_demo['dataset'] = 'train'\ndemo_combined = train_demo.copy()\n\n# --- Unified plotting function\ndef create_categorical_plots(variable, data, source_name):\n sns.set_style('whitegrid')\n fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n\n # Pie Chart (left)\n plt.subplot(1, 2, 1)\n value_counts = data[variable].value_counts()\n\n # Collapse small categories\n threshold = 0.05 * value_counts.sum()\n filtered_values = value_counts.copy()\n filtered_values[value_counts < threshold] = 0\n filtered_values = filtered_values[filtered_values > 0]\n other_count = value_counts.sum() - filtered_values.sum()\n if other_count > 0:\n filtered_values['Other'] = other_count\n\n wedges, texts, autotexts = plt.pie(\n filtered_values,\n autopct=lambda p: f'{p:.1f}%' if p > 5 else '',\n colors=pie_chart_palette[:len(filtered_values)],\n startangle=140,\n wedgeprops=dict(width=0.3),\n explode=[0.05 if p > 5 else 0 for p in filtered_values],\n textprops={'fontsize': 10}\n )\n\n plt.title(\"\\n\".join(textwrap.wrap(f\"Pie Chart for {variable} — {source_name}\", width=50)))\n plt.legend(filtered_values.index, loc=\"upper left\", bbox_to_anchor=(1, 1))\n\n # Horizontal Countplot (right)\n plt.subplot(1, 2, 2)\n sns.countplot(\n data=data,\n y=variable,\n hue='dataset', # still present, but only \"train\"\n palette=custom_palette,\n alpha=0.85\n )\n plt.ylabel(variable)\n plt.xlabel(\"Count\")\n plt.title(\"\\n\".join(textwrap.wrap(f\"Countplot for {variable} — {source_name}\", width=50)))\n plt.tight_layout()\n plt.show()\n\n# --- Plot all categorical variables from main dataset (train only)\nfor var in categorical_variables:\n if var in main_combined.columns:\n create_categorical_plots(var, main_combined, \"Context Features\")\n\n# --- Plot all categorical variables from demographic dataset (train only)\nfor var in categorical_variables:\n if var in demo_combined.columns:\n create_categorical_plots(var, demo_combined, \"Demographic Features\")\n\n# Cleanup\ntrain_main.drop('dataset', axis=1, inplace=True)\ntrain_demo.drop('dataset', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:04.862458Z","iopub.execute_input":"2025-08-07T02:02:04.862921Z","iopub.status.idle":"2025-08-07T02:02:12.232717Z","shell.execute_reply.started":"2025-08-07T02:02:04.862895Z","shell.execute_reply":"2025-08-07T02:02:12.231754Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3 Target Feature Analysis (Univariate Analysis)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport textwrap\n\n# Define palettes\npie_chart_palette = ['#33638d', '#28ae80', '#d3eb0c', '#ff9a0b', '#7e03a8', '#35b779',\n '#fde725', '#440154', '#90d743', '#482173', '#22a884', '#f8961e']\n\n# Define target column(s)\ntarget_columns = ['gesture']\n\n# Prepare the main data (ONLY train set)\ntrain_main = train_df.copy()\ntrain_main['dataset'] = 'train' # kept if you want to add back more logic later\n\n# Unified plotting function for target variables (train only)\ndef create_target_plots(variable, data, source_name):\n sns.set_style('whitegrid')\n fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n\n # Pie Chart (left)\n plt.subplot(1, 2, 1)\n value_counts = data[variable].value_counts()\n\n # Collapse small categories into 'Other'\n threshold = 0.05 * value_counts.sum()\n filtered_values = value_counts.copy()\n filtered_values[value_counts < threshold] = 0\n filtered_values = filtered_values[filtered_values > 0]\n other_count = value_counts.sum() - filtered_values.sum()\n if other_count > 0:\n filtered_values['Other'] = other_count\n\n wedges, texts, autotexts = plt.pie(\n filtered_values,\n autopct=lambda p: f'{p:.1f}%' if p > 5 else '',\n colors=pie_chart_palette[:len(filtered_values)],\n startangle=140,\n wedgeprops=dict(width=0.3),\n explode=[0.05 if p > 5 else 0 for p in filtered_values],\n textprops={'fontsize': 10}\n )\n\n plt.title(\"\\n\".join(textwrap.wrap(f\"Pie Chart for {variable} — {source_name}\", width=50)))\n plt.legend(filtered_values.index, loc=\"upper left\", bbox_to_anchor=(1, 1))\n\n # Horizontal Countplot (right)\n plt.subplot(1, 2, 2)\n sns.countplot(\n data=data,\n y=variable,\n color='#3498db', # just use a single color (train)\n alpha=0.85\n )\n plt.ylabel(variable)\n plt.xlabel(\"Count\")\n plt.title(\"\\n\".join(textwrap.wrap(f\"Countplot for {variable} — {source_name}\", width=50)))\n plt.tight_layout()\n plt.show()\n\n# Plot target column(s) (train only)\nfor var in target_columns:\n if var in train_main.columns:\n create_target_plots(var, train_main, \"Target Variable\")\n\n# Cleanup\ntrain_main.drop('dataset', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:12.234058Z","iopub.execute_input":"2025-08-07T02:02:12.234376Z","iopub.status.idle":"2025-08-07T02:02:14.409035Z","shell.execute_reply.started":"2025-08-07T02:02:12.234354Z","shell.execute_reply":"2025-08-07T02:02:14.408031Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.4 Multivariate Feature Relationship","metadata":{}},{"cell_type":"markdown","source":"### 3.4.1 Sensor Feature Analysis for all Gestures (Single Random Subject)","metadata":{}},{"cell_type":"code","source":"#fix code dòng dưới\nsubject_id = train_dem_df['subject'].sample(1).iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:14.410314Z","iopub.execute_input":"2025-08-07T02:02:14.410590Z","iopub.status.idle":"2025-08-07T02:02:14.415965Z","shell.execute_reply.started":"2025-08-07T02:02:14.410570Z","shell.execute_reply":"2025-08-07T02:02:14.415183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"subj_info = train_dem_df[train_dem_df['subject'] == subject_id].iloc[0]\n\nprint(f\"⏩ Details of the random subject being considered\")\nprint(f\"Subject ID: {subject_id}\")\nprint(f\" • Age: {subj_info['age']} years\")\nprint(f\" • Adult/Child: {'Adult' if subj_info['adult_child']==1 else 'Child'}\")\nprint(f\" • Sex: {'Male' if subj_info['sex']==1 else 'Female'}\")\nprint(f\" • Handedness: {'Right-handed' if subj_info['handedness']==1 else 'Left-handed'}\")\nprint(f\" • Height: {subj_info['height_cm']} cm\")\nprint(f\" • Shoulder→Wrist: {subj_info['shoulder_to_wrist_cm']} cm\")\nprint(f\" • Elbow→Wrist: {subj_info['elbow_to_wrist_cm']} cm\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:14.416909Z","iopub.execute_input":"2025-08-07T02:02:14.417200Z","iopub.status.idle":"2025-08-07T02:02:14.439537Z","shell.execute_reply.started":"2025-08-07T02:02:14.417170Z","shell.execute_reply":"2025-08-07T02:02:14.438601Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### ⏩ 1. ** BIỂU ĐỒ IMU**\n#### **📌 Hướng dẫn giải thích**\n * Độ lớn gia tốc (`acc_mag`): cho biết cổ tay di chuyển “mạnh” như thế nào. Các đỉnh thường tương ứng với chuyển động tay nhanh.\n * Góc xoay (`rot_angle`): đại diện cho mức độ cổ tay bị vặn lệch so với tư thế trung tính ban đầu. Những cú nhảy đột ngột có thể cho thấy các động tác lật cổ tay nhanh.\n * Vùng được tô xám “Transition” thể hiện khoảng thời gian tay đang di chuyển tới mục tiêu thực hiện cử chỉ; vùng màu đỏ “Gesture” thể hiện khoảng thời gian tiếp xúc thực sự hoặc động tác chính diễn ra.\n * So sánh hai đường biểu diễn này trong từng chuỗi dữ liệu giúp phân biệt, ví dụ, một hành động BFRB như giật tóc (gia tốc cao kèm góc xoay đột ngột) với một chuyển động nhẹ nhàng không phải BFRB.","metadata":{}},{"cell_type":"code","source":"# ─── Cell 1: IMU Time Series Plots ───────────────────────────────────────────────────────────────────────────────────────\n# This cell computes the acceleration magnitude (√(acc_x²+acc_y²+acc_z²)) and the rotation angle (θ = 2·arccos(rot_w))\n# for each timestamp in a chosen subject’s sequences. It then plots, for each gesture sequence, two lines:\n# • acc_mag (acceleration magnitude)\n# • rot_angle (orientation angle)\n# The background is shaded by ‘phase’: “Transition” vs. “Gesture” in different colors.\n# Each subplot title shows the gesture name and its sequence_type. Two plots are arranged per row.\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n# Select one subject (e.g., first in the list)\nsubject_id = train_df['subject'].unique()[0]\nsubj_df = train_df[train_df['subject'] == subject_id].copy()\n\n# Compute derived IMU features\nsubj_df['acc_mag'] = np.sqrt(\n subj_df['acc_x']**2 + subj_df['acc_y']**2 + subj_df['acc_z']**2\n)\nsubj_df['rot_w_clipped'] = subj_df['rot_w'].clip(-1, 1)\nsubj_df['rot_angle'] = 2 * np.arccos(subj_df['rot_w_clipped'])\n\n# Pick one sequence_id per unique gesture label\n# Group by the integer-encoded 'gesture' column, then take the first sequence_id in each group\ngesture_to_seq = subj_df.groupby('gesture')['sequence_id'].first().to_dict()\n# Convert to a list of sequence_ids\nseq_ids = list(gesture_to_seq.values())\n\nn = len(seq_ids)\nncols = 2\nnrows = int(np.ceil(n / ncols))\n\nfig, axes = plt.subplots(nrows, ncols, figsize=(14, 4 * nrows), sharex=False, sharey=False)\naxes = axes.flatten()\n\nfor i, seq in enumerate(seq_ids):\n ax = axes[i]\n seq_df = subj_df[subj_df['sequence_id'] == seq].sort_values('sequence_counter')\n times = seq_df['sequence_counter']\n\n # Plot the two derived IMU lines\n ax.plot(times, seq_df['acc_mag'],\n label='Acceleration Magnitude', color='tab:blue', linewidth=1.5)\n ax.plot(times, seq_df['rot_angle'],\n label='Rotation Angle (rad)', color='tab:orange', linewidth=1.5)\n\n # Shade by phase: “Transition” vs “Gesture”\n used_labels = set()\n for phase_label, color in [('Transition', 'lightgray'),\n ('Gesture', 'lightcoral')]:\n mask = seq_df['phase'] == phase_label\n if mask.any():\n # Find contiguous intervals where phase == phase_label\n idxs = seq_df.index[mask]\n breaks = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start_idx = idxs[0]\n for b in breaks:\n end_idx = idxs[b]\n spans.append((start_idx, end_idx))\n start_idx = idxs[b + 1]\n spans.append((start_idx, idxs[-1]))\n # Draw each span; label only once per phase in this subplot\n for (start_i, end_i) in spans:\n t0 = seq_df.loc[start_i, 'sequence_counter']\n t1 = seq_df.loc[end_i, 'sequence_counter']\n label_arg = phase_label if phase_label not in used_labels else None\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=label_arg)\n used_labels.add(phase_label)\n\n # Subplot titles and labels\n gesture_name = seq_df['gesture'].iloc[0]\n seq_type = seq_df['sequence_type'].iloc[0]\n ax.set_title(f\"Subject {subject_id} – Seq {seq} – {gesture_name} ({seq_type})\",\n fontsize=10)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Value\", fontsize=9)\n ax.legend(loc='upper right', fontsize='small')\n ax.grid(True)\n\n# Turn off any empty subplots\nfor j in range(i + 1, nrows * ncols):\n fig.delaxes(axes[j])\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:14.440552Z","iopub.execute_input":"2025-08-07T02:02:14.440879Z","iopub.status.idle":"2025-08-07T02:02:19.908279Z","shell.execute_reply.started":"2025-08-07T02:02:14.440848Z","shell.execute_reply":"2025-08-07T02:02:19.907206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### ⏩ 2. ** Biểu đồ Thermopile**\n#### **📌 Hướng dẫn phân tích**\n* Mỗi đường cong `thm_i` đại diện cho nhiệt độ gần da đo được từ 5 cảm biến. Khi giá trị thm_i tăng lên, điều đó cho thấy cổ tay đang tiến gần đến một bộ phận ấm trên cơ thể (ví dụ: mặt, cổ).\n\n* Đường đậm “`Average Temperature`” là trung bình của cả 5 cảm biến, giúp làm rõ các sự kiện tiếp xúc (đỉnh nhiệt độ tăng) so với các giai đoạn ở xa (vùng nhiệt độ thấp, phẳng).\n\n* Vùng được tô màu theo “Transition” và “Gesture” giúp xác nhận rằng sự gia tăng nhiệt độ chủ yếu diễn ra trong giai đoạn “Gesture” (tức là khi có tiếp xúc thực sự).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Use the same subject_id and seq_ids (one per unique gesture) from Cell 1\ntherm_df = subj_df.copy()\n\n# Gather thermopile columns and compute their per-timestamp mean\nthm_cols = [f'thm_{i}' for i in range(1, 6)]\ntherm_df['thm_mean'] = therm_df[thm_cols].mean(axis=1)\n\nfig, axes = plt.subplots(nrows, ncols, figsize=(14, 4 * nrows), sharex=False, sharey=False)\naxes = axes.flatten()\n\nfor i, seq in enumerate(seq_ids):\n ax = axes[i]\n seq_thm = therm_df[therm_df['sequence_id'] == seq].sort_values('sequence_counter')\n times = seq_thm['sequence_counter']\n # Determine y-range for shading (not strictly needed since axvspan spans entire y-axis)\n # Plot each thermopile channel\n for col in thm_cols:\n ax.plot(times, seq_thm[col], label=col, linewidth=1)\n # Plot the average as a bold black line\n ax.plot(times, seq_thm['thm_mean'], label='Average Temperature', color='black', linewidth=2.5)\n # Shade by phase using axvspan\n for phase_label, color in [('Transition', 'lightgray'), ('Gesture', 'lightcoral')]:\n mask = seq_thm['phase'] == phase_label\n if mask.any():\n idxs = seq_thm.index[mask]\n breaks = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start_idx = idxs[0]\n for b in breaks:\n end_idx = idxs[b]\n spans.append((start_idx, end_idx))\n start_idx = idxs[b + 1]\n spans.append((start_idx, idxs[-1]))\n for (start_i, end_i) in spans:\n t0 = seq_thm.loc[start_i, 'sequence_counter']\n t1 = seq_thm.loc[end_i, 'sequence_counter']\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=phase_label)\n # Labels and legend for this subplot\n gesture_name = seq_thm['gesture'].iloc[0]\n seq_type = seq_thm['sequence_type'].iloc[0]\n ax.set_title(f\"Subject {subject_id} – Seq {seq} – {gesture_name} ({seq_type})\", fontsize=10)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Temperature (°C)\", fontsize=9)\n ax.legend(loc='upper right', fontsize='small')\n ax.grid(True)\n\n# Turn off any unused subplots\nfor j in range(i + 1, nrows * ncols):\n fig.delaxes(axes[j])\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:19.909538Z","iopub.execute_input":"2025-08-07T02:02:19.909858Z","iopub.status.idle":"2025-08-07T02:02:26.200110Z","shell.execute_reply.started":"2025-08-07T02:02:19.909833Z","shell.execute_reply":"2025-08-07T02:02:26.199024Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### ⏩ 3. **ToF Plots**\n#### **📌 Interpretation Guide**\n * Mỗi giá trị `tof_i_mean` là trung bình của lưới cảm biến 8×8, thể hiện mức độ “gần” của cổ tay so với một vật thể (ví dụ: da). Giá trị thấp (\\~0–50) → rất gần; giá trị cao hơn (\\~100–200) → xa hơn; giá trị –1 (bị che/mất tín hiệu) → không có tín hiệu phản hồi.\n * Theo dõi giá trị trung bình của cả năm cảm biến theo thời gian giúp xác định thời điểm tay tiến lại gần (giá trị đột ngột giảm) và khi tay rời xa.\n * Phần tô màu theo “Transition” và “Gesture” xác nhận rằng khoảng cách thấp nhất trùng với thời điểm tiếp xúc thực sự (diễn ra trong giai đoạn “Gesture”).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Use the same subject_id and seq_ids (one per unique gesture) from Cell 1\ntof_df = subj_df.copy()\n\n# Identify the 64 columns for each ToF sensor and compute their per-timestamp mean\nmean_cols = []\nfor i_sensor in range(1, 6):\n pixel_cols = [f'tof_{i_sensor}_v{pix}' for pix in range(64)]\n tof_df[f'tof_{i_sensor}_mean'] = tof_df[pixel_cols].replace(-1, np.nan).mean(axis=1)\n mean_cols.append(f'tof_{i_sensor}_mean')\n\nfig, axes = plt.subplots(nrows, ncols, figsize=(14, 4 * nrows), sharex=False, sharey=False)\naxes = axes.flatten()\n\nfor i, seq in enumerate(seq_ids):\n ax = axes[i]\n seq_tof = tof_df[tof_df['sequence_id'] == seq].sort_values('sequence_counter')\n times = seq_tof['sequence_counter']\n\n # Plot mean of each ToF sensor\n for col in mean_cols:\n ax.plot(times, seq_tof[col], label=col, linewidth=1)\n\n # Shade by phase using axvspan\n for phase_label, color in [('Transition', 'lightgray'), ('Gesture', 'lightcoral')]:\n mask = seq_tof['phase'] == phase_label\n if mask.any():\n idxs = seq_tof.index[mask]\n breaks = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start_idx = idxs[0]\n for b in breaks:\n end_idx = idxs[b]\n spans.append((start_idx, end_idx))\n start_idx = idxs[b + 1]\n spans.append((start_idx, idxs[-1]))\n for (start_i, end_i) in spans:\n t0 = seq_tof.loc[start_i, 'sequence_counter']\n t1 = seq_tof.loc[end_i, 'sequence_counter']\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=phase_label)\n\n # Labels and legend for this subplot\n gesture_name = seq_tof['gesture'].iloc[0]\n seq_type = seq_tof['sequence_type'].iloc[0]\n ax.set_title(f\"Subject {subject_id} – Seq {seq} – {gesture_name} ({seq_type})\", fontsize=10)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Mean ToF Distance\", fontsize=9)\n ax.legend(loc='upper right', fontsize='small')\n ax.grid(True)\n\n# Turn off any unused subplots\nfor j in range(i + 1, nrows * ncols):\n fig.delaxes(axes[j])\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:26.201860Z","iopub.execute_input":"2025-08-07T02:02:26.202308Z","iopub.status.idle":"2025-08-07T02:02:32.371891Z","shell.execute_reply.started":"2025-08-07T02:02:26.202270Z","shell.execute_reply":"2025-08-07T02:02:32.370613Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.4.2 Sensor‐Based Differentiation Across Demographics and Physiology Features","metadata":{}},{"cell_type":"markdown","source":"#### 📌 Đặc trưng đang xem xét: (Thuận tay) **Handedness**\n\nTrong phần này, ta trực quan hóa cách những người thuận tay trái và thuận tay phải thực hiện cùng một cử chỉ (“Write name on leg”). Ta lấy một chuỗi dữ liệu từ mỗi nhóm và vẽ song song bốn loại cảm biến để so sánh.\n\nBằng cách so sánh giữa người thuận tay trái và tay phải, ta tìm kiếm sự khác biệt ổn định trong cách các cảm biến trên cổ tay phản hồi khi tay tiến lại gần và viết lên chân. Do các cảm biến Helios được bố trí xung quanh cổ tay, việc thuận tay nào sẽ ảnh hưởng đến cảm biến thermopile hoặc ToF nào đối diện với chân trước, đồng thời làm thay đổi hướng chuyển động trên các trục IMU.\n\n**⏩ Cách phân tích & Hàm ý thực tiễn (theo từng cảm biến):**\n\n* **IMU (acc\\_x/y/z):** Quan sát trục gia tốc nào có đỉnh sớm hơn hoặc lớn hơn khi tay di chuyển vào trong. Một người thuận tay phải khi viết có thể tạo ra giá trị acc_x (lateral – ngang) dương lớn hơn do cổ tay xoay khác với tay trái. Mô hình có thể học cách hoán đổi hoặc căn chỉnh lại thông tin trục dựa vào handedness để tránh hiểu nhầm thời điểm bắt đầu cử chỉ.\n* **Rotation (rot\\_w/x/y/z):** Theo dõi xu hướng nghiêng: người thuận tay phải có thể xoay cổ tay vào trong (ví dụ rot_y tăng sớm hơn), còn tay trái có thể xoay ra ngoài trước. Nếu góc xoay (rot_angle) tăng chậm hơn ở một bên, điều đó cho biết cánh tay vặn khác nhau — cần điều chỉnh căn thời gian để tránh nhầm lẫn do handedness.\n* **Thermopiles (thm\\_1–5):** Để ý cảm biến nhiệt nào tăng trước khi nhiệt da lọt vào vùng nhìn 50°. Với người thuận tay phải, một cảm biến cụ thể (ví dụ thm_2 ở mặt trong cổ tay) có thể nhận tín hiệu nhiệt sớm hơn so với tay trái. Nhờ đó, thuật toán có thể chọn cảm biến nhiệt phù hợp dựa trên thông tin handedness khai báo.\n* **ToF Mean Distance (tof\\_1–5):** Xác định cảm biến ToF nào có khoảng cách trung bình giảm sớm nhất khi tay tiếp cận chân. Người thuận tay phải có thể khiến ToF mặt trước (ví dụ tof_3_mean) nhận biết khoảng cách gần sớm hơn, và điều này ngược lại với người thuận tay trái. Việc huấn luyện trên sự lệch thời gian có hệ thống của ToF theo handedness giúp mô hình tránh bị overfitting vào một kiểu tay cụ thể.\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# (1) Merge train_demographics into train_df if not already done\ntrain_df = train_df.merge(\n train_dem_df,\n on=\"subject\",\n how=\"left\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:32.373351Z","iopub.execute_input":"2025-08-07T02:02:32.373685Z","iopub.status.idle":"2025-08-07T02:02:33.045313Z","shell.execute_reply.started":"2025-08-07T02:02:32.373662Z","shell.execute_reply":"2025-08-07T02:02:33.044301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"⏩ Vẽ sơ đồ cho mọi đối tượng chỉ với một cử chỉ","metadata":{}},{"cell_type":"code","source":"import warnings, numpy as np, pandas as pd, seaborn as sns, matplotlib.pyplot as plt\nwarnings.filterwarnings('ignore')\n\n# ─── 1. Choose grouping variable ───────────────────────────────────────────────────────\ngroup_col = \"handedness\"\ngroup_map = {0: \"Left-handed\", 1: \"Right-handed\"}\n\n# ─── 2. Build sequence-level summary table ─────────────────────────────────────────────\nseq_summaries = []\n\nfor seq_id, seq in train_df.groupby(\"sequence_id\"):\n \n g_label = seq[group_col].iloc[0]\n if pd.isna(g_label): # skip if demographic missing\n continue\n \n # ----- Phase masks -----\n gest_mask = seq[\"phase\"] == \"Gesture\"\n \n # ----- 1) Acceleration jerk -----\n # jerk = |Δacc_mag / Δt| ; use max over entire seq\n acc_mag = np.sqrt(seq[\"acc_x\"]**2 + seq[\"acc_y\"]**2 + seq[\"acc_z\"]**2).values\n jerk = np.abs(np.diff(acc_mag)) # Δt = 1 frame\n max_jerk = jerk.max() if len(jerk) else np.nan\n \n # ----- 2) Rotation angular range (Gesture phase only) -----\n rot_angle = 2*np.arccos(seq[\"rot_w\"].clip(-1,1))\n rot_range = rot_angle[gest_mask].max() - rot_angle[gest_mask].min() if gest_mask.any() else np.nan\n \n # ----- 3) Thermopile peak ΔT (Gesture peak minus Transition baseline) -----\n thm_cols = [f\"thm_{i}\" for i in range(1,6)]\n thm_mean = seq[thm_cols].ffill().bfill().mean(axis=1)\n baseline = thm_mean[seq[\"phase\"]==\"Transition\"].median()\n peak_dt = thm_mean[gest_mask].max() - baseline if gest_mask.any() else np.nan\n \n # ----- 4) ToF closest mean distance (min over all 5 sensors) -----\n tof_means = []\n for i in range(1,6):\n col = f\"tof_{i}_mean\"\n if col not in seq:\n # compute on the fly (replace -1 by NaN)\n pix = seq[[f\"tof_{i}_v{p}\" for p in range(64)]].replace(-1, np.nan)\n seq[col] = pix.mean(axis=1)\n tof_means.append(seq[col])\n tof_overall = pd.concat(tof_means, axis=1).mean(axis=1)\n min_tof = tof_overall.min()\n \n seq_summaries.append({\n \"sequence_id\": seq_id,\n group_col: g_label,\n \"max_jerk\": max_jerk,\n \"rot_range\": rot_range,\n \"thm_peak_delta\": peak_dt,\n \"tof_min_distance\": min_tof\n })\n\nsummary_df = pd.DataFrame(seq_summaries).dropna()\n\n# Map group labels to readable strings\nsummary_df[\"Group\"] = summary_df[group_col].map(group_map)\n\n# ─── 3. Plot distributions ────────────────────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(2, 2, figsize=(14, 10))\nplt.subplots_adjust(hspace=0.35, wspace=0.25)\n\nmetrics = [\n (\"max_jerk\", \"Max Jerk |Δacc| (m/s²)\"),\n (\"rot_range\", \"Gesture Rot-Angle Range (rad)\"),\n (\"thm_peak_delta\", \"Thermopile ΔT Peak (°C)\"),\n (\"tof_min_distance\", \"Min ToF Mean Dist. (units)\")\n]\n\nfor ax, (metric, nice_name) in zip(axes.flatten(), metrics):\n sns.violinplot(\n data=summary_df,\n x=\"Group\", y=metric, palette=\"Set2\",\n inner=None, ax=ax\n )\n sns.boxplot(\n data=summary_df,\n x=\"Group\", y=metric,\n width=0.2, showcaps=True, boxprops={'facecolor':'white'},\n showfliers=False, whiskerprops={'linewidth':2},\n ax=ax\n )\n ax.set_title(nice_name, fontsize=12)\n ax.set_xlabel(\"\")\n ax.set_ylabel(nice_name, fontsize=10)\n\nfig.suptitle(f\"{group_map[0]} vs. {group_map[1]} – Sequence-Level Sensor Features (*Gesture Phase Only)\", fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:02:33.046370Z","iopub.execute_input":"2025-08-07T02:02:33.046682Z","iopub.status.idle":"2025-08-07T02:04:04.337698Z","shell.execute_reply.started":"2025-08-07T02:02:33.046651Z","shell.execute_reply":"2025-08-07T02:04:04.336659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# ─── Compute per-sequence phase durations ────────────────────────────────────────────────────────────\n# Assumes train_df already has demographics merged and 'phase' column\n\n# Count number of frames per phase, per sequence\ndur_df = (\n train_df\n .groupby(['sequence_id', 'handedness'])['phase']\n .value_counts()\n .unstack(fill_value=0)\n .reset_index()\n)\n\n# Keep only Transition & Gesture (in case Pause exists)\ndur_df = dur_df[['sequence_id', 'handedness', 'Transition', 'Gesture']]\n\n# Melt to long form for boxplot\ndur_long = dur_df.melt(\n id_vars=['sequence_id', 'handedness'],\n value_vars=['Transition', 'Gesture'],\n var_name='phase',\n value_name='duration_frames'\n)\n\n# Map handedness to labels\ndur_long['Group'] = dur_long['handedness'].map({0: 'Left-handed', 1: 'Right-handed'})\n\n# ─── Plot horizontal boxplots side-by-side ───────────────────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(1, 2, figsize=(14, 6), sharey=True)\nplt.subplots_adjust(wspace=0.2)\n\nfor ax, group_label in zip(axes, ['Left-handed', 'Right-handed']):\n subset = dur_long[dur_long['Group'] == group_label]\n sns.boxplot(\n data=subset,\n x='duration_frames',\n y='phase',\n orient='h',\n palette=['lightgray', 'lightcoral'],\n ax=ax\n )\n ax.set_title(f\"{group_label}\", fontsize=12)\n ax.set_xlabel(\"Duration (frames)\", fontsize=10)\n ax.set_ylabel(\"Phase\", fontsize=10)\n ax.grid(True)\n\nfig.suptitle(\"Phase Durations by Handedness: Transition vs. Gesture\", fontsize=14, y=1.02)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:04:04.338874Z","iopub.execute_input":"2025-08-07T02:04:04.339234Z","iopub.status.idle":"2025-08-07T02:04:04.965318Z","shell.execute_reply.started":"2025-08-07T02:04:04.339193Z","shell.execute_reply":"2025-08-07T02:04:04.964407Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"⏩ Biểu đồ cho các đối tượng được chọn ngẫu nhiên và một cử chỉ","metadata":{}},{"cell_type":"code","source":"# ─── Cell A: Handedness Comparison for IMU, Rot, Thermopile, and ToF ────────────────────────────────────────────\n\n# (2) Pick one gesture to compare (change as needed)\ngesture_to_plot = \"Write name on leg\"\n\n# (3) Filter to only rows with that gesture, and split by handedness\ndf_gesture = train_df[train_df[\"gesture\"] == gesture_to_plot].copy()\n\n# Left‐handed group (handedness == 0) and Right‐handed group (handedness == 1)\nleft_group = df_gesture[df_gesture[\"handedness\"] == 0]\nright_group = df_gesture[df_gesture[\"handedness\"] == 1]\n\n# (4) From each group, pick a single sequence_id at random (or first) for plotting\n# If no left‐handed example exists, you may need another gesture.\nif left_group[\"sequence_id\"].nunique() == 0:\n raise ValueError(\"No left‐handed example of that gesture found. Choose a different gesture.\")\n\nleft_seq = left_group[\"sequence_id\"].unique()[0]\nright_seq = right_group[\"sequence_id\"].unique()[0]\n\n# (5) Extract the two sequences, sort by sequence_counter\ndf_left = train_df[(train_df[\"sequence_id\"] == left_seq)].sort_values(\"sequence_counter\")\ndf_right = train_df[(train_df[\"sequence_id\"] == right_seq)].sort_values(\"sequence_counter\")\n\n# (6) Compute IMU derived features: acc_mag and rot_angle\nfor df in (df_left, df_right):\n df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n df[\"rot_w_clipped\"] = df[\"rot_w\"].clip(-1, 1)\n df[\"rot_angle\"] = 2 * np.arccos(df[\"rot_w_clipped\"])\n\n# (7) Plotting helper to shade phases\ndef shade_phases(ax, seq_df):\n used = set()\n for phase_label, color in [(\"Transition\", \"lightgray\"), (\"Gesture\", \"lightcoral\")]:\n mask = seq_df[\"phase\"] == phase_label\n if not mask.any():\n continue\n idxs = seq_df.index[mask]\n diffs = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start = idxs[0]\n for b in diffs:\n end = idxs[b]\n spans.append((start, end))\n start = idxs[b + 1]\n spans.append((start, idxs[-1]))\n for (s, e) in spans:\n t0 = seq_df.loc[s, \"sequence_counter\"]\n t1 = seq_df.loc[e, \"sequence_counter\"]\n label = phase_label if phase_label not in used else None\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=label)\n used.add(phase_label)\n\n# (8) Now produce **four** separate figures (each with 2 subplots side‐by‐side):\n\n# -----------------------------------------------------------------------------\n# A.1: IMU: Acceleration Magnitude\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_left, df_right],\n [f\"Left‐handed (Subj: {df_left['subject'].iloc[0]})\",\n f\"Right‐handed (Subj: {df_right['subject'].iloc[0]})\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_mag\"],\n color=\"tab:blue\", lw=1.5, label=\"|acc|(m/s²)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Acceleration Magnitude\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Handedness: Acceleration Magnitude – Left vs. Right\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# A.2: IMU: Rotation Angle\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_left, df_right],\n [f\"Left‐handed (Subj: {df_left['subject'].iloc[0]})\",\n f\"Right‐handed (Subj: {df_right['subject'].iloc[0]})\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"rot_angle\"],\n color=\"tab:orange\", lw=1.5, label=\"rot_angle (rad)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Rotation Angle (rad)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Handedness: Rotation Angle – Left vs. Right\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# A.3: Thermopile: 5 Channels + Average\n# -----------------------------------------------------------------------------\n# (Compute thermopile mean if not already done)\nfor df in (df_left, df_right):\n thm_cols = [f\"thm_{i}\" for i in range(1,6)]\n df[\"thm_mean\"] = df[thm_cols].mean(axis=1)\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_left, df_right],\n [f\"Left‐handed (Subj: {df_left['subject'].iloc[0]})\",\n f\"Right‐handed (Subj: {df_right['subject'].iloc[0]})\"]\n):\n times = df_seq[\"sequence_counter\"]\n # Plot the five raw channels\n for col in [f\"thm_{i}\" for i in range(1,6)]:\n ax.plot(times, df_seq[col], lw=1, label=col, alpha=0.7)\n # Plot the average as bold\n ax.plot(times, df_seq[\"thm_mean\"], color=\"black\", lw=2.5, label=\"thm_mean\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Thermopile (°C)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"x-small\", ncol=2)\n ax.grid(True)\nplt.suptitle(\"Handedness: Thermopile – Left vs. Right\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# A.4: ToF: Five Mean‐Distance Channels\n# -----------------------------------------------------------------------------\n# (Compute ToF mean per‐sensor)\nfor df in (df_left, df_right):\n for i_sensor in range(1,6):\n pixel_cols = [f\"tof_{i_sensor}_v{pix}\" for pix in range(64)]\n df[f\"tof_{i_sensor}_mean\"] = df[pixel_cols].replace(-1, np.nan).mean(axis=1)\n\nmean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_left, df_right],\n [f\"Left‐handed (Subj: {df_left['subject'].iloc[0]})\",\n f\"Right‐handed (Subj: {df_right['subject'].iloc[0]})\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in mean_cols:\n ax.plot(times, df_seq[col], lw=1, label=col)\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"ToF Mean Distance\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Handedness: ToF Mean Distance – Left vs. Right\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:04:04.966380Z","iopub.execute_input":"2025-08-07T02:04:04.966656Z","iopub.status.idle":"2025-08-07T02:04:07.862573Z","shell.execute_reply.started":"2025-08-07T02:04:04.966626Z","shell.execute_reply":"2025-08-07T02:04:07.861569Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"📌 Đặc trưng đang xem xét: Adult vs. Child\nỞ đây, ta chọn một cử chỉ cụ thể (ví dụ: “Neck – pinch skin”) và so sánh giữa dữ liệu của một người lớn và một trẻ em.\n\nMỗi biểu đồ đối chiếu cách trẻ em và người lớn thực hiện cùng một cử chỉ. Trẻ em thường di chuyển đột ngột hơn (các đỉnh “acc_mag” cao hơn) và có thể tạo ra các đặc điểm nhiệt/tiếp cận khác do chiều dài tay ngắn hơn.\n\n⏩ Cách phân tích & Hàm ý thực tiễn (theo từng cảm biến):\n\nAcceleration magnitude: Nếu đường cong “acc_mag” của trẻ có nhiều đỉnh sắc nhọn và biên độ lớn hơn, mô hình cần hiểu rằng các đợt gia tốc IMU mạnh không phải lúc nào cũng là dấu hiệu của BFRB, mà có thể phản ánh sự vận động tự nhiên, mạnh mẽ hơn của trẻ em.\nRotation angle: Người lớn có thể giữ ổn định góc xoay hơn trong giai đoạn tạm dừng, trong khi “rot_angle” của trẻ có thể dao động nhiều hơn. Điều này giúp mô hình nhận biết rằng những dao động nhỏ có thể là bình thường nếu đối tượng là trẻ em.\nThermopile: Giá trị nhiệt của trẻ có thể tăng nhanh hơn (tiếp xúc gần hơn) nhưng cũng hạ nhiệt nhanh hơn; tích hợp đặc điểm này giúp mô hình phân biệt tiếp xúc thực sự và các pha bật ra nhanh điển hình ở trẻ em.\nToF: Nếu khoảng cách trung bình của ToF ở trẻ giảm sớm hơn (tay ở gần hơn), mô hình có thể học cách điều chỉnh ngưỡng khoảng cách theo độ tuổi, tránh nhầm lẫn các chuyển động lớn của trẻ với những cử chỉ khác.","metadata":{}},{"cell_type":"markdown","source":"⏩ Plots for All Subjects and One Gesture","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# ─── 1. Build sequence-level summaries for Adult vs. Child ─────────────────────────\nseq_summaries = []\nfor seq_id, seq in train_df.groupby(\"sequence_id\"):\n grp = seq[\"adult_child\"].iloc[0] # 0=child, 1=adult\n # Skip if no phase labels\n if seq[\"phase\"].isna().all():\n continue\n # 1) Max Jerk (IMU)\n acc_mag = np.sqrt(seq[\"acc_x\"]**2 + seq[\"acc_y\"]**2 + seq[\"acc_z\"]**2).values\n jerk = np.abs(np.diff(acc_mag))\n max_jerk = jerk.max() if len(jerk) else np.nan\n # 2) Rotation range during Gesture\n rot_angle = 2 * np.arccos(seq[\"rot_w\"].clip(-1,1))\n mask_g = seq[\"phase\"]==\"Gesture\"\n rot_range = rot_angle[mask_g].max() - rot_angle[mask_g].min() if mask_g.any() else np.nan\n # 3) Thermopile peak ΔT\n thm = seq[[f\"thm_{i}\" for i in range(1,6)]].ffill().bfill().mean(axis=1)\n base = thm[seq[\"phase\"]==\"Transition\"].median()\n peak_dt = thm[mask_g].max() - base if mask_g.any() else np.nan\n # 4) ToF min distance\n tof_means = []\n for i in range(1,6):\n col = f\"tof_{i}_mean\"\n if col not in seq:\n pix = seq[[f\"tof_{i}_v{p}\" for p in range(64)]].replace(-1,np.nan)\n seq[col] = pix.mean(axis=1)\n tof_means.append(seq[col])\n tof_min = pd.concat(tof_means,axis=1).mean(axis=1).min()\n seq_summaries.append({\n \"sequence_id\": seq_id,\n \"adult_child\": grp,\n \"max_jerk\": max_jerk,\n \"rot_range\": rot_range,\n \"thm_peak_delta\": peak_dt,\n \"tof_min_distance\": tof_min\n })\n\nsummary_df = pd.DataFrame(seq_summaries).dropna()\nsummary_df[\"Group\"] = summary_df[\"adult_child\"].map({0:\"Child\", 1:\"Adult\"})\n\n# ─── 2. Plot distributions side-by-side ───────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(2, 2, figsize=(14,10), sharey=False)\nplt.subplots_adjust(hspace=0.4, wspace=0.3)\n\nmetrics = [\n (\"max_jerk\", \"Max Jerk |Δacc| (m/s²)\"),\n (\"rot_range\", \"Gesture Rot-Range (rad)\"),\n (\"thm_peak_delta\", \"Thermopile ΔT Peak (°C)\"),\n (\"tof_min_distance\", \"Min ToF Mean Dist. (units)\")\n]\n\nfor ax, (col, label) in zip(axes.flatten(), metrics):\n sns.violinplot(\n x=\"Group\", y=col, data=summary_df,\n palette=\"Set2\", inner=None, ax=ax\n )\n sns.boxplot(\n x=\"Group\", y=col, data=summary_df,\n width=0.2, showcaps=True, boxprops={'facecolor':'white'},\n showfliers=False, whiskerprops={'linewidth':2}, ax=ax\n )\n ax.set_title(label, fontsize=12)\n ax.set_xlabel(\"\")\n ax.set_ylabel(label, fontsize=10)\n\nfig.suptitle(\"Adult vs. Child – Sequence-Level Sensor Feature Distributions (*Gesture Phase Only)\", fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:04:07.863461Z","iopub.execute_input":"2025-08-07T02:04:07.863701Z","iopub.status.idle":"2025-08-07T02:05:40.733390Z","shell.execute_reply.started":"2025-08-07T02:04:07.863683Z","shell.execute_reply":"2025-08-07T02:05:40.732428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nwarnings.filterwarnings('ignore')\nsns.set_style(\"whitegrid\")\n\n# ─── Compute per-sequence phase durations for Adult vs. Child ─────────────────────────────────────────────────\ndur_df = (\n train_df\n .groupby(['sequence_id', 'adult_child'])['phase']\n .value_counts()\n .unstack(fill_value=0)\n .reset_index()\n)\n\n# Keep only Transition & Gesture columns\ndur_df = dur_df[['sequence_id', 'adult_child', 'Transition', 'Gesture']]\n\n# Melt to long form\ndur_long = dur_df.melt(\n id_vars=['sequence_id', 'adult_child'],\n value_vars=['Transition', 'Gesture'],\n var_name='phase',\n value_name='duration_frames'\n)\n\n# Map adult_child to labels\ndur_long['Group'] = dur_long['adult_child'].map({0: 'Child', 1: 'Adult'})\n\n# ─── Plot side-by-side horizontal boxplots ─────────────────────────────────────────────────────────────────────\nfig, axes = plt.subplots(1, 2, figsize=(14, 6), sharey=True)\nplt.subplots_adjust(wspace=0.3)\n\nfor ax, group_label in zip(axes, ['Child', 'Adult']):\n subset = dur_long[dur_long['Group'] == group_label]\n sns.boxplot(\n data=subset,\n x='duration_frames',\n y='phase',\n orient='h',\n palette=['lightgray', 'lightcoral'],\n ax=ax\n )\n ax.set_title(f\"{group_label}\", fontsize=12)\n ax.set_xlabel(\"Duration (frames)\", fontsize=10)\n ax.set_ylabel(\"Phase\", fontsize=10)\n ax.grid(True)\n\nplt.suptitle(\"Phase Durations by Adult vs. Child\", fontsize=14, y=1.02)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:05:40.734492Z","iopub.execute_input":"2025-08-07T02:05:40.734758Z","iopub.status.idle":"2025-08-07T02:05:41.354320Z","shell.execute_reply.started":"2025-08-07T02:05:40.734736Z","shell.execute_reply":"2025-08-07T02:05:41.353426Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"⏩ Biểu đồ cho các đối tượng được chọn ngẫu nhiên và một cử chỉ","metadata":{}},{"cell_type":"code","source":"# ─── Cell B: Adult vs. Child Comparison for IMU, Rot, Thermopile, and ToF ───────────────────────────────────────\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n# (2) Pick one gesture to compare (change as needed)\ngesture_to_plot = \"Neck - pinch skin\"\n\n# (3) Filter to only rows with that gesture, split by adult_child (0=child, 1=adult)\ndf_gesture = train_df[train_df[\"gesture\"] == gesture_to_plot].copy()\nchild_group = df_gesture[df_gesture[\"adult_child\"] == 0]\nadult_group = df_gesture[df_gesture[\"adult_child\"] == 1]\n\nif (child_group[\"sequence_id\"].nunique() == 0) or (adult_group[\"sequence_id\"].nunique() == 0):\n raise ValueError(\"Insufficient examples in one of the groups. Pick another gesture.\")\n\nchild_seq = child_group[\"sequence_id\"].unique()[0]\nadult_seq = adult_group[\"sequence_id\"].unique()[0]\n\ndf_child = train_df[train_df[\"sequence_id\"] == child_seq].sort_values(\"sequence_counter\")\ndf_adult = train_df[train_df[\"sequence_id\"] == adult_seq].sort_values(\"sequence_counter\")\n\n# Compute derived IMU features on each\nfor df in (df_child, df_adult):\n df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n df[\"rot_w_clipped\"] = df[\"rot_w\"].clip(-1, 1)\n df[\"rot_angle\"] = 2 * np.arccos(df[\"rot_w_clipped\"])\n\n# Plot Helper to shade phases (re‐use function from Cell A)\ndef shade_phases(ax, seq_df):\n used = set()\n for phase_label, color in [(\"Transition\", \"lightgray\"), (\"Gesture\", \"lightcoral\")]:\n mask = seq_df[\"phase\"] == phase_label\n if not mask.any():\n continue\n idxs = seq_df.index[mask]\n diffs = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start = idxs[0]\n for b in diffs:\n end = idxs[b]\n spans.append((start, end))\n start = idxs[b + 1]\n spans.append((start, idxs[-1]))\n for (s, e) in spans:\n t0 = seq_df.loc[s, \"sequence_counter\"]\n t1 = seq_df.loc[e, \"sequence_counter\"]\n label = phase_label if phase_label not in used else None\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=label)\n used.add(phase_label)\n\n# -----------------------------------------------------------------------------\n# B.1: IMU – Acceleration Magnitude\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_child, df_adult],\n [f\"Child (Subj: {df_child['subject'].iloc[0]}, Age: {df_child['age'].iloc[0]})\",\n f\"Adult (Subj: {df_adult['subject'].iloc[0]}, Age: {df_adult['age'].iloc[0]})\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_mag\"],\n color=\"tab:blue\", lw=1.5, label=\"|acc|(m/s²)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Acc Magnitude\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Adult vs. Child: Acceleration Magnitude\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# B.2: IMU – Rotation Angle\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_child, df_adult],\n [f\"Child (Subj: {df_child['subject'].iloc[0]}, Age: {df_child['age'].iloc[0]})\",\n f\"Adult (Subj: {df_adult['subject'].iloc[0]}, Age: {df_adult['age'].iloc[0]})\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"rot_angle\"],\n color=\"tab:orange\", lw=1.5, label=\"rot_angle (rad)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Rot Angle (rad)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Adult vs. Child: Rotation Angle\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# B.3: Thermopile – 5 Channels + Mean\n# -----------------------------------------------------------------------------\nfor df in (df_child, df_adult):\n thm_cols = [f\"thm_{i}\" for i in range(1,6)]\n df[\"thm_mean\"] = df[thm_cols].mean(axis=1)\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_child, df_adult],\n [f\"Child (Subj: {df_child['subject'].iloc[0]}, Age: {df_child['age'].iloc[0]})\",\n f\"Adult (Subj: {df_adult['subject'].iloc[0]}, Age: {df_adult['age'].iloc[0]})\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in [f\"thm_{i}\" for i in range(1,6)]:\n ax.plot(times, df_seq[col], lw=1, label=col, alpha=0.7)\n ax.plot(times, df_seq[\"thm_mean\"], color=\"black\", lw=2.5, label=\"thm_mean\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Thermopile (°C)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\", ncol=2)\n ax.grid(True)\nplt.suptitle(\"Adult vs. Child: Thermopile\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# B.4: ToF – Five Mean‐Distance Channels\n# -----------------------------------------------------------------------------\nfor df in (df_child, df_adult):\n for i_sensor in range(1,6):\n pixel_cols = [f\"tof_{i_sensor}_v{pix}\" for pix in range(64)]\n df[f\"tof_{i_sensor}_mean\"] = df[pixel_cols].replace(-1, np.nan).mean(axis=1)\n\nmean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_child, df_adult],\n [f\"Child (Subj: {df_child['subject'].iloc[0]}, Age: {df_child['age'].iloc[0]})\",\n f\"Adult (Subj: {df_adult['subject'].iloc[0]}, Age: {df_adult['age'].iloc[0]})\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in mean_cols:\n ax.plot(times, df_seq[col], lw=1, label=col)\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"ToF Mean Distance\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Adult vs. Child: ToF Mean Distance\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:05:41.355410Z","iopub.execute_input":"2025-08-07T02:05:41.355712Z","iopub.status.idle":"2025-08-07T02:05:44.532038Z","shell.execute_reply.started":"2025-08-07T02:05:41.355678Z","shell.execute_reply":"2025-08-07T02:05:44.531014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 📌 Đặc trưng đang xem xét: **Sex (Female vs. Male)**\n\nTrong phần này, ta so sánh các tín hiệu cảm biến giữa nam và nữ đối với một cử chỉ cụ thể (ví dụ: “Forehead – scratch”). Ta tạo cùng lúc bốn biểu đồ: IMU acc_mag, IMU rot_angle, Thermopile, và ToF. Những khác biệt có thể xuất hiện do sự chênh lệch về sức mạnh cơ bắp trung bình, kích thước bàn tay hoặc phong cách thực hiện cử chỉ.\n\n**⏩ HCách phân tích & Hàm ý thực tiễn:**\n\n* **IMU – Acceleration:** Nếu nam giới thể hiện các đỉnh gia tốc cao hơn một cách nhất quán (do sức mạnh cơ bắp trung bình lớn hơn), mô hình có thể chuẩn hóa cường độ IMU hoặc học cách hiệu chỉnh theo giới tính.\n* **Rotation Angle:**Nam và nữ có thể có kiểu xoay cổ tay khác nhau; bộ phân loại không nên quá phụ thuộc vào một mẫu rot_angle nhất định mà bỏ qua yếu tố giới tính.\n* **Thermopile:** Nhiệt độ da cơ bản của nữ thường khác nhẹ so với nam, do đó các đường nhiệt có thể bắt đầu ở các giá trị khác nhau. Mô hình cần học cách tập trung vào sự thay đổi tương đối (relative changes) thay vì tuyệt đối, từ đó tăng độ ổn định khi dự đoán.\n* **ToF:** Hình dạng bàn tay của nam có thể tạo ra các cụm điểm ToF với kích thước khác biệt. Việc đưa giới tính vào mô hình giúp điều chỉnh ngưỡng khoảng cách và mô hình hóa không gian tiếp cận, giảm thiểu các lỗi phân loại khi đánh giá mức độ gần của tay đối với cơ thể.\n\n\n\n\n\n\n\n\n\n","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# ─── 1. Choose grouping variable for this cell ───────────────────────────────────────────────────────────────────\ngroup_col = \"sex\"\ngroup_map = {0: \"Female\", 1: \"Male\"}\n\n# ─── 2. Build sequence-level summaries (if not yet created) ─────────────────────────────────────────────────────\n# Reuse logic from the previous overview cell, but group on 'sex' instead of 'handedness'.\n\nseq_stats = []\nfor seq_id, seq in train_df.groupby(\"sequence_id\"):\n g = seq[group_col].iloc[0]\n if pd.isna(g):\n continue\n\n # Phase masks\n gest = seq[\"phase\"] == \"Gesture\"\n\n # 1) IMU jerk\n acc = np.sqrt(seq[\"acc_x\"]**2 + seq[\"acc_y\"]**2 + seq[\"acc_z\"]**2).values\n max_jerk = np.abs(np.diff(acc)).max() if len(acc)>1 else np.nan\n\n # 2) Rotation range in gesture\n rot = 2*np.arccos(seq[\"rot_w\"].clip(-1,1))\n rot_range = rot[gest].max() - rot[gest].min() if gest.any() else np.nan\n\n # 3) Thermopile peak delta\n thms = seq[[f\"thm_{i}\" for i in range(1,6)]].ffill().bfill().mean(axis=1)\n baseline = thms[seq[\"phase\"]==\"Transition\"].median() if (seq[\"phase\"]==\"Transition\").any() else np.nan\n thm_peak = thms[gest].max() - baseline if gest.any() else np.nan\n\n # 4) ToF min mean distance\n tcols = []\n for i in range(1,6):\n col = f\"tof_{i}_mean\"\n if col not in seq:\n pix = seq[[f\"tof_{i}_v{p}\" for p in range(64)]].replace(-1, np.nan)\n seq[col] = pix.mean(axis=1)\n tcols.append(seq[col])\n tof_all = pd.concat(tcols, axis=1).mean(axis=1)\n tof_min = tof_all.min()\n\n seq_stats.append({\n \"sequence_id\": seq_id,\n group_col: g,\n \"max_jerk\": max_jerk,\n \"rot_range\": rot_range,\n \"thm_peak_delta\": thm_peak,\n \"tof_min_distance\": tof_min\n })\n\nsummary = pd.DataFrame(seq_stats).dropna()\nsummary[\"Group\"] = summary[group_col].map(group_map)\n\n# ─── 3. Plot tailored distributions ───────────────────────────────────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(2, 2, figsize=(14, 10))\nplt.subplots_adjust(hspace=0.35, wspace=0.25)\n\nmetrics = [\n (\"max_jerk\", \"Max Jerk |Δacc| (m/s²)\"),\n (\"rot_range\", \"Rotation Range (rad)\"),\n (\"thm_peak_delta\", \"Thermopile ΔT Peak (°C)\"),\n (\"tof_min_distance\", \"Min ToF Mean Dist.\")\n]\n\nfor ax, (col, lbl) in zip(axes.flatten(), metrics):\n sns.violinplot(\n data=summary,\n x=\"Group\", y=col,\n palette=\"Set2\",\n inner=None,\n ax=ax\n )\n sns.boxplot(\n data=summary,\n x=\"Group\", y=col,\n width=0.2,\n showcaps=True,\n boxprops={'facecolor':'white'},\n showfliers=False,\n whiskerprops={'linewidth':2},\n ax=ax\n )\n ax.set_title(lbl, fontsize=12)\n ax.set_xlabel(\"\")\n ax.set_ylabel(lbl, fontsize=10)\n\nfig.suptitle(\"Sex Comparison: Sequence‐Level Sensor Summaries (*Gesture Phase Only)\", fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:05:44.533003Z","iopub.execute_input":"2025-08-07T02:05:44.533305Z","iopub.status.idle":"2025-08-07T02:07:16.994354Z","shell.execute_reply.started":"2025-08-07T02:05:44.533282Z","shell.execute_reply":"2025-08-07T02:07:16.993347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# ─── Compute per-sequence phase durations for Sex comparison ───────────────────────────────────────────────────────\n# Assumes train_df already merged with train_dem_df\n\n# Group by sequence_id and sex, count frames in each phase\ndur_df_sex = (\n train_df\n .groupby(['sequence_id', 'sex'])['phase']\n .value_counts()\n .unstack(fill_value=0)\n .reset_index()\n)\n\n# Keep only Transition & Gesture\ndur_df_sex = dur_df_sex[['sequence_id', 'sex', 'Transition', 'Gesture']]\n\n# Melt to long form\ndur_long_sex = dur_df_sex.melt(\n id_vars=['sequence_id', 'sex'],\n value_vars=['Transition', 'Gesture'],\n var_name='phase',\n value_name='duration_frames'\n)\n\n# Map sex to labels\ndur_long_sex['Group'] = dur_long_sex['sex'].map({0: 'Female', 1: 'Male'})\n\n# ─── Plot horizontal boxplots side-by-side ─────────────────────────────────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(1, 2, figsize=(14, 6), sharey=True)\nplt.subplots_adjust(wspace=0.2)\n\nfor ax, group_label in zip(axes, ['Female', 'Male']):\n subset = dur_long_sex[dur_long_sex['Group'] == group_label]\n sns.boxplot(\n data=subset,\n x='duration_frames',\n y='phase',\n orient='h',\n palette={'Transition':'lightgray', 'Gesture':'lightcoral'},\n ax=ax\n )\n ax.set_title(f\"{group_label}\", fontsize=12)\n ax.set_xlabel(\"Duration (frames)\", fontsize=10)\n ax.set_ylabel(\"Phase\", fontsize=10)\n ax.grid(True)\n\nfig.suptitle(\"Sex Comparison: Transition vs. Gesture Duration\", fontsize=14, y=1.02)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:16.995447Z","iopub.execute_input":"2025-08-07T02:07:16.995740Z","iopub.status.idle":"2025-08-07T02:07:17.576160Z","shell.execute_reply.started":"2025-08-07T02:07:16.995714Z","shell.execute_reply":"2025-08-07T02:07:17.575293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─── Cell C: Sex Comparison for IMU, Rot, Thermopile, and ToF ───────────────────────────────────────────────────\n\n# (2) Pick one gesture to compare (change as needed)\ngesture_to_plot = \"Forehead - scratch\"\n\n# (3) Filter to only rows with that gesture, split by sex (0=female, 1=male)\ndf_gesture = train_df[train_df[\"gesture\"] == gesture_to_plot].copy()\nfemale_group = df_gesture[df_gesture[\"sex\"] == 0]\nmale_group = df_gesture[df_gesture[\"sex\"] == 1]\n\nif (female_group[\"sequence_id\"].nunique() == 0) or (male_group[\"sequence_id\"].nunique() == 0):\n raise ValueError(\"Not enough examples for each sex. Try a different gesture.\")\n\nfemale_seq = female_group[\"sequence_id\"].unique()[0]\nmale_seq = male_group[\"sequence_id\"].unique()[0]\n\ndf_fem = train_df[train_df[\"sequence_id\"] == female_seq].sort_values(\"sequence_counter\")\ndf_male= train_df[train_df[\"sequence_id\"] == male_seq].sort_values(\"sequence_counter\")\n\n# Compute derived IMU features\nfor df in (df_fem, df_male):\n df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n df[\"rot_w_clipped\"] = df[\"rot_w\"].clip(-1, 1)\n df[\"rot_angle\"] = 2 * np.arccos(df[\"rot_w_clipped\"])\n\n# Re‐use shading function\ndef shade_phases(ax, seq_df):\n used = set()\n for phase_label, color in [(\"Transition\", \"lightgray\"), (\"Gesture\", \"lightcoral\")]:\n mask = seq_df[\"phase\"] == phase_label\n if not mask.any():\n continue\n idxs = seq_df.index[mask]\n diffs = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start = idxs[0]\n for b in diffs:\n end = idxs[b]\n spans.append((start, end))\n start = idxs[b + 1]\n spans.append((start, idxs[-1]))\n for (s, e) in spans:\n t0 = seq_df.loc[s, \"sequence_counter\"]\n t1 = seq_df.loc[e, \"sequence_counter\"]\n label = phase_label if phase_label not in used else None\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=label)\n used.add(phase_label)\n\n# -----------------------------------------------------------------------------\n# C.1: IMU – Acceleration Magnitude\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_fem, df_male],\n [f\"Female (Subj: {df_fem['subject'].iloc[0]}, Sex=0)\",\n f\"Male (Subj: {df_male['subject'].iloc[0]}, Sex=1)\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_mag\"],\n color=\"tab:blue\", lw=1.5, label=\"|acc|(m/s²)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Acceleration Magnitude\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Sex: Acceleration Magnitude – Female vs. Male\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# C.2: IMU – Rotation Angle\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_fem, df_male],\n [f\"Female (Subj: {df_fem['subject'].iloc[0]}, Sex=0)\",\n f\"Male (Subj: {df_male['subject'].iloc[0]}, Sex=1)\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"rot_angle\"],\n color=\"tab:orange\", lw=1.5, label=\"rot_angle (rad)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Rotation Angle (rad)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Sex: Rotation Angle – Female vs. Male\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# C.3: Thermopile – 5 Channels + Mean\n# -----------------------------------------------------------------------------\nfor df in (df_fem, df_male):\n thm_cols = [f\"thm_{i}\" for i in range(1,6)]\n df[\"thm_mean\"] = df[thm_cols].mean(axis=1)\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_fem, df_male],\n [f\"Female (Subj: {df_fem['subject'].iloc[0]}, Sex=0)\",\n f\"Male (Subj: {df_male['subject'].iloc[0]}, Sex=1)\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in [f\"thm_{i}\" for i in range(1,6)]:\n ax.plot(times, df_seq[col], lw=1, label=col, alpha=0.7)\n ax.plot(times, df_seq[\"thm_mean\"], color=\"black\", lw=2.5, label=\"thm_mean\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Thermopile (°C)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\", ncol=2)\n ax.grid(True)\nplt.suptitle(\"Sex: Thermopile – Female vs. Male\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# C.4: ToF – Five Mean‐Distance Channels\n# -----------------------------------------------------------------------------\nfor df in (df_fem, df_male):\n for i_sensor in range(1,6):\n pixel_cols = [f\"tof_{i_sensor}_v{pix}\" for pix in range(64)]\n df[f\"tof_{i_sensor}_mean\"] = df[pixel_cols].replace(-1, np.nan).mean(axis=1)\n\nmean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_fem, df_male],\n [f\"Female (Subj: {df_fem['subject'].iloc[0]}, Sex=0)\",\n f\"Male (Subj: {df_male['subject'].iloc[0]}, Sex=1)\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in mean_cols:\n ax.plot(times, df_seq[col], lw=1, label=col)\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"ToF Mean Distance\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Sex: ToF Mean Distance – Female vs. Male\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:17.577201Z","iopub.execute_input":"2025-08-07T02:07:17.577460Z","iopub.status.idle":"2025-08-07T02:07:20.601202Z","shell.execute_reply.started":"2025-08-07T02:07:17.577440Z","shell.execute_reply":"2025-08-07T02:07:20.600388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 📌 Đặc trưng đang xem xét: **Chiều dài từ vai đến cổ tay**\n\nTrong phần này, dữ liệu được tách theo nhóm chiều dài tay (tay ngắn vs. tay dài) cho một cử chỉ cụ thể (ví dụ: “Eyelash – pull hair”). Ta trực quan hóa cùng lúc bốn tín hiệu cảm biến. Người có cánh tay dài hơn có thể tiếp cận khuôn mặt với góc độ và tốc độ khác, từ đó ảnh hưởng tới toàn bộ dấu hiệu từ cảm biến.\n\n**⏩ How to Interpret & Implications:**\n\n* **IMU:** Những người có tay dài thường có đồ thị acc_mag mượt hơn (ít thay đổi đột ngột hơn) vì cánh tay dài tạo ra gia tốc góc chậm hơn. Mô hình nên học cách chuẩn hóa theo chiều dài tay để tránh nhầm lẫn giữa độ mượt của chuyển động với việc không thực hiện cử chỉ mạnh.\n* **Rotation Angle:** Người tay dài có thể có đỉnh rot_angle thấp hơn (do quỹ đạo lớn hơn nhưng tốc độ xoay chậm hơn). Bộ lọc thời gian của mô hình cần điều chỉnh cho phù hợp, tránh hiểu nhầm xoay chậm thành cử chỉ khác.\n* **Thermopile:** Tay dài có thể khiến các cảm biến nhiệt phát hiện nhiệt muộn hơn (do cổ tay phải di chuyển xa hơn mới tiếp cận được bề mặt cơ thể). Việc huấn luyện mô hình với cả hai nhóm chiều dài tay giúp tránh việc nhầm lẫn sự trễ này thành không có tiếp xúc.\n* **ToF:** Đối với người tay dài, khoảng cách ToF bắt đầu giảm ở giá trị xa hơn (tay di chuyển qua nhiều không gian hơn trước khi chạm vào mục tiêu). Mô hình cần học cách đặt ngưỡng khoảng cách phù hợp theo chiều dài tay để giảm lỗi bỏ sót khi tiếp xúc diễn ra ở xa hơn.","metadata":{}},{"cell_type":"markdown","source":"⏩ Plots for All Subjects and One Gesture","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# ─── Sequence‐Level Summaries for Short vs. Long Forearm ───────────────────────────────────────────────────────────\n# We define 4 tailored metrics as before, but group by\n# 'short' = shoulder_to_wrist_cm ≤ median\n# 'long' = shoulder_to_wrist_cm > median\n\n# 1) Choose gesture (same as plotting cell)\ngesture_to_plot = \"Eyelash - pull hair\"\ndf_g = train_df[train_df[\"gesture\"] == gesture_to_plot].copy()\n\n# 2) Determine median forearm length\nmedian_len = df_g[\"shoulder_to_wrist_cm\"].median()\n\n# 3) Build summaries\nrecords = []\nfor seq_id, seq in train_df[train_df[\"gesture\"] == gesture_to_plot].groupby(\"sequence_id\"):\n L = seq[\"shoulder_to_wrist_cm\"].iloc[0]\n grp = \"Short\" if L <= median_len else \"Long\"\n # IMU jerk\n acc_mag = np.sqrt(seq[\"acc_x\"]**2 + seq[\"acc_y\"]**2 + seq[\"acc_z\"]**2)\n jerks = np.abs(np.diff(acc_mag))\n max_jerk = jerks.max() if len(jerks)>0 else np.nan\n # Rotational range in Gesture\n rot = 2*np.arccos(seq[\"rot_w\"].clip(-1,1))\n rot_rng = rot[seq[\"phase\"]==\"Gesture\"].max() - rot[seq[\"phase\"]==\"Gesture\"].min() \\\n if (seq[\"phase\"]==\"Gesture\").any() else np.nan\n # Thermopile peak delta\n thm_cols = [f\"thm_{i}\" for i in range(1,6)]\n thm_mean = seq[thm_cols].mean(axis=1)\n base = thm_mean[seq[\"phase\"]==\"Transition\"].median()\n peak_dt = thm_mean[seq[\"phase\"]==\"Gesture\"].max() - base \\\n if (seq[\"phase\"]==\"Gesture\").any() else np.nan\n # ToF min mean\n tof_means = []\n for i in range(1,6):\n pix = seq[[f\"tof_{i}_v{p}\" for p in range(64)]].replace(-1,np.nan)\n tof_means.append(pix.mean(axis=1))\n tof_all = pd.concat(tof_means,axis=1).mean(axis=1)\n min_tof = tof_all.min()\n records.append({\n \"Group\": grp,\n \"max_jerk\": max_jerk,\n \"rot_range\": rot_rng,\n \"thm_peak_dt\": peak_dt,\n \"tof_min_dist\": min_tof\n })\n\nsummary = pd.DataFrame(records).dropna()\n\n# ─── Plot tailored violin+box for each metric ───────────────────────────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(2, 2, figsize=(14,10))\nplt.subplots_adjust(hspace=0.4, wspace=0.3)\n\nmetrics = [\n (\"max_jerk\", \"Max Jerk |Δacc| (m/s²)\"),\n (\"rot_range\", \"Rotation Range in Gesture (rad)\"),\n (\"thm_peak_dt\", \"Thermopile Peak ΔT (°C)\"),\n (\"tof_min_dist\", \"Minimum ToF Mean Dist.\")\n]\n\nfor ax, (col, label) in zip(axes.flatten(), metrics):\n sns.violinplot(data=summary, x=\"Group\", y=col, palette=\"Set2\", ax=ax, inner=None)\n sns.boxplot(data=summary, x=\"Group\", y=col,\n width=0.2, showcaps=True, boxprops={'facecolor':'white'},\n showfliers=False, whiskerprops={'linewidth':2}, ax=ax)\n ax.set_title(label, fontsize=12)\n ax.set_xlabel(\"\")\n ax.set_ylabel(label, fontsize=10)\n\nfig.suptitle(f\"Forearm Length Comparison (≤{median_len:.1f}cm vs >{median_len:.1f}cm) (*Gesture Phase Only)\", fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:20.602391Z","iopub.execute_input":"2025-08-07T02:07:20.603115Z","iopub.status.idle":"2025-08-07T02:07:27.750590Z","shell.execute_reply.started":"2025-08-07T02:07:20.603065Z","shell.execute_reply":"2025-08-07T02:07:27.749614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ─── Compute per-sequence phase durations for all sequences ────────────────────────────────────────────────\n# Assumes train_df already has shoulder_to_wrist_cm merged and 'phase' column\n\n# Tag each sequence as Short vs. Long arm based on median shoulder_to_wrist_cm across all sequences\nmed = train_df.groupby('sequence_id')['shoulder_to_wrist_cm'].first().median()\nseq_meta = (\n train_df\n .groupby('sequence_id')\n .agg({'shoulder_to_wrist_cm':'first'})\n .reset_index()\n)\nseq_meta['ArmGroup'] = np.where(seq_meta['shoulder_to_wrist_cm'] <= med,\n 'Short Arm', 'Long Arm')\n\n# Count frames per phase per sequence\ndur = (\n train_df\n .groupby(['sequence_id','phase'])['sequence_counter']\n .count()\n .unstack(fill_value=0)\n .reset_index()\n)\n\n# Merge in arm group label\ndur = dur.merge(seq_meta[['sequence_id','ArmGroup']], on='sequence_id')\n\n# Melt to long form\ndur_long = dur.melt(id_vars=['sequence_id','ArmGroup'],\n value_vars=['Transition','Gesture'],\n var_name='Phase', value_name='Duration')\n\n# ─── Plot Horizontal Boxplots: Short vs. Long Arm ─────────────────────────────────────────────────────────\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(1, 2, figsize=(14,5), sharey=True)\nfor ax, grp in zip(axes, ['Short Arm','Long Arm']):\n sub = dur_long[dur_long['ArmGroup']==grp]\n sns.boxplot(x='Duration', y='Phase', data=sub,\n palette=['lightgray','lightcoral'], ax=ax)\n ax.set_title(f\"{grp} (≤ {med:.1f} cm)\", fontsize=12)\n ax.set_xlabel(\"Duration (frames)\", fontsize=10)\n ax.set_ylabel(\"Phase\", fontsize=10)\n ax.grid(True)\n\nplt.suptitle(\"Phase Durations by Arm Length Group\", fontsize=14, y=1.05)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:27.751601Z","iopub.execute_input":"2025-08-07T02:07:27.752178Z","iopub.status.idle":"2025-08-07T02:07:28.624665Z","shell.execute_reply.started":"2025-08-07T02:07:27.752150Z","shell.execute_reply":"2025-08-07T02:07:28.623782Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"⏩ Plots for Randomly Selected Subjects and One Gesture","metadata":{}},{"cell_type":"code","source":"# ─── Cell D: Shoulder‐to‐Wrist Length Comparison (Short vs. Long) for IMU, Rot, Thermopile, and ToF ───────────\n\n# (2) Pick one gesture to compare (change as needed)\ngesture_to_plot = \"Eyelash - pull hair\"\n\n# (3) Filter to that gesture and split by shoulder_to_wrist length using median\ndf_gesture = train_df[train_df[\"gesture\"] == gesture_to_plot].copy()\nmedian_shoulder = df_gesture[\"shoulder_to_wrist_cm\"].median()\n\nshort_group = df_gesture[df_gesture[\"shoulder_to_wrist_cm\"] <= median_shoulder]\nlong_group = df_gesture[df_gesture[\"shoulder_to_wrist_cm\"] > median_shoulder]\n\nif (short_group[\"sequence_id\"].nunique() == 0) or (long_group[\"sequence_id\"].nunique() == 0):\n raise ValueError(\"Not enough examples in one of the length‐groups. Try another gesture or adjust threshold.\")\n\nshort_seq = short_group[\"sequence_id\"].unique()[0]\nlong_seq = long_group[\"sequence_id\"].unique()[0]\n\ndf_short = train_df[train_df[\"sequence_id\"] == short_seq].sort_values(\"sequence_counter\")\ndf_long = train_df[train_df[\"sequence_id\"] == long_seq].sort_values(\"sequence_counter\")\n\n# Compute IMU derived features\nfor df in (df_short, df_long):\n df[\"acc_mag\"] = np.sqrt(df[\"acc_x\"]**2 + df[\"acc_y\"]**2 + df[\"acc_z\"]**2)\n df[\"rot_w_clipped\"] = df[\"rot_w\"].clip(-1, 1)\n df[\"rot_angle\"] = 2 * np.arccos(df[\"rot_w_clipped\"])\n\n# Re‐use shading function\ndef shade_phases(ax, seq_df):\n used = set()\n for phase_label, color in [(\"Transition\", \"lightgray\"), (\"Gesture\", \"lightcoral\")]:\n mask = seq_df[\"phase\"] == phase_label\n if not mask.any():\n continue\n idxs = seq_df.index[mask]\n diffs = np.where(np.diff(idxs) != 1)[0]\n spans = []\n start = idxs[0]\n for b in diffs:\n end = idxs[b]\n spans.append((start, end))\n start = idxs[b + 1]\n spans.append((start, idxs[-1]))\n for (s, e) in spans:\n t0 = seq_df.loc[s, \"sequence_counter\"]\n t1 = seq_df.loc[e, \"sequence_counter\"]\n label = phase_label if phase_label not in used else None\n ax.axvspan(t0, t1, color=color, alpha=0.3, label=label)\n used.add(phase_label)\n\n# -----------------------------------------------------------------------------\n# D.1: IMU – Acceleration Magnitude\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_short, df_long],\n [f\"Short Arm (Shoulder‐Wrist ≤ {median_shoulder:.1f} cm)\",\n f\"Long Arm (Shoulder‐Wrist > {median_shoulder:.1f} cm)\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_mag\"],\n color=\"tab:blue\", lw=1.5, label=\"|acc|(m/s²)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Acceleration Magnitude\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Arm Length: Acceleration Magnitude – Short vs. Long\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# D.2: IMU – Rotation Angle\n# -----------------------------------------------------------------------------\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_short, df_long],\n [f\"Short Arm (Shoulder‐Wrist ≤ {median_shoulder:.1f} cm)\",\n f\"Long Arm (Shoulder‐Wrist > {median_shoulder:.1f} cm)\"]\n):\n ax.plot(df_seq[\"sequence_counter\"], df_seq[\"rot_angle\"],\n color=\"tab:orange\", lw=1.5, label=\"rot_angle (rad)\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Rotation Angle (rad)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Arm Length: Rotation Angle – Short vs. Long\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# D.3: Thermopile – 5 Channels + Mean\n# -----------------------------------------------------------------------------\nfor df in (df_short, df_long):\n thm_cols = [f\"thm_{i}\" for i in range(1,6)]\n df[\"thm_mean\"] = df[thm_cols].mean(axis=1)\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_short, df_long],\n [f\"Short Arm (Shoulder‐Wrist ≤ {median_shoulder:.1f} cm)\",\n f\"Long Arm (Shoulder‐Wrist > {median_shoulder:.1f} cm)\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in [f\"thm_{i}\" for i in range(1,6)]:\n ax.plot(times, df_seq[col], lw=1, label=col, alpha=0.7)\n ax.plot(times, df_seq[\"thm_mean\"], color=\"black\", lw=2.5, label=\"thm_mean\")\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"Thermopile (°C)\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\", ncol=2)\n ax.grid(True)\nplt.suptitle(\"Arm Length: Thermopile – Short vs. Long\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()\n\n# -----------------------------------------------------------------------------\n# D.4: ToF – Five Mean‐Distance Channels\n# -----------------------------------------------------------------------------\nfor df in (df_short, df_long):\n for i_sensor in range(1,6):\n pixel_cols = [f\"tof_{i_sensor}_v{pix}\" for pix in range(64)]\n df[f\"tof_{i_sensor}_mean\"] = df[pixel_cols].replace(-1, np.nan).mean(axis=1)\n\nmean_cols = [f\"tof_{i}_mean\" for i in range(1,6)]\nfig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)\nfor ax, df_seq, title in zip(\n axes,\n [df_short, df_long],\n [f\"Short Arm (Shoulder‐Wrist ≤ {median_shoulder:.1f} cm)\",\n f\"Long Arm (Shoulder‐Wrist > {median_shoulder:.1f} cm)\"]\n):\n times = df_seq[\"sequence_counter\"]\n for col in mean_cols:\n ax.plot(times, df_seq[col], lw=1, label=col)\n shade_phases(ax, df_seq)\n ax.set_title(f\"{title}\\nGesture: {gesture_to_plot}\", fontsize=11)\n ax.set_xlabel(\"Sequence Counter\", fontsize=9)\n ax.set_ylabel(\"ToF Mean Distance\", fontsize=9)\n ax.legend(loc=\"upper right\", fontsize=\"small\")\n ax.grid(True)\nplt.suptitle(\"Arm Length: ToF Mean Distance – Short vs. Long\", fontsize=13)\nplt.tight_layout(rect=[0, 0, 1, 0.95])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:28.625763Z","iopub.execute_input":"2025-08-07T02:07:28.626118Z","iopub.status.idle":"2025-08-07T02:07:31.631809Z","shell.execute_reply.started":"2025-08-07T02:07:28.626069Z","shell.execute_reply":"2025-08-07T02:07:31.630838Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.3.3 Correlation analysis","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\n\n# Combine train & test for both datasets\ncombined_main = pd.concat([train_df, test_df], axis=0)\ncombined_demo = pd.concat([train_dem_df, test_dem_df], axis=0)\n\n# Set target\ntarget_variable = 'gesture'\n\n# Define sensor prefixes and exclusion list\nsensor_prefixes = ('acc_', 'rot_', 'thm_', 'tof_')\nexcluded_columns = [target_variable]\n\n# Identify usable numerical variables (non-sensor, numeric only)\ndef get_numerical_columns(df, exclude_prefixes, excluded_cols):\n return [col for col in df.columns \n if pd.api.types.is_numeric_dtype(df[col]) \n and not col.startswith(exclude_prefixes)\n and col not in excluded_cols]\n\nmain_vars = get_numerical_columns(combined_main, sensor_prefixes, excluded_columns)\ndemo_vars = get_numerical_columns(combined_demo, (), excluded_columns)\n\n# Combine both for global correlation\ncombined_data = pd.concat([\n combined_main[main_vars].reset_index(drop=True),\n combined_demo[demo_vars].reset_index(drop=True)\n], axis=1)\n\n# Compute correlation and mask upper triangle\ncorr_all = combined_data.corr()\nmask_all = np.triu(np.ones_like(corr_all, dtype=bool))\n\n# Plot\nplt.figure(figsize=(18, 10))\nax = sns.heatmap(\n corr_all, mask=mask_all, cmap='viridis', annot=True, \n square=False, linewidths=.5, annot_kws={\"size\": 12}\n)\nplt.title('Correlation Heatmap — Combined (Demographic + Main) Data', fontsize=16)\n\n# Rotate x-axis tick labels\nax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha='right', fontsize=11)\nax.set_yticklabels(ax.get_yticklabels(), fontsize=11)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:31.632799Z","iopub.execute_input":"2025-08-07T02:07:31.633061Z","iopub.status.idle":"2025-08-07T02:07:33.183327Z","shell.execute_reply.started":"2025-08-07T02:07:31.633040Z","shell.execute_reply":"2025-08-07T02:07:33.182306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Copy and encode target\ntrain_corr_df = train_df.copy()\ntrain_corr_df[target_variable] = LabelEncoder().fit_transform(train_corr_df[target_variable])\n\n# Extract valid numerical features from train only\ntrain_main_vars = get_numerical_columns(train_corr_df, sensor_prefixes, excluded_columns)\n\n# Include demographic features too\ntrain_demo_vars = get_numerical_columns(train_dem_df, (), excluded_columns)\n\n# Combine both\ntrain_all_corr = pd.concat([train_corr_df[train_main_vars], train_dem_df[train_demo_vars],\n train_corr_df[[target_variable]]], axis=1)\n\n# Correlation with target only\ncorr_target_only = train_all_corr.corr()[[target_variable]].T\n\n# Plot\nplt.figure(figsize=(12, 3))\nsns.heatmap(corr_target_only, cmap='viridis', annot=True, linewidths=0.5, cbar=False, annot_kws={\"size\": 10})\nplt.title(\"Correlation with Target (gesture) — Train Data\", fontsize=13)\nplt.xticks(rotation=45, ha='right')\nplt.yticks(rotation=0)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T02:07:33.184777Z","iopub.execute_input":"2025-08-07T02:07:33.185124Z","iopub.status.idle":"2025-08-07T02:07:35.913822Z","shell.execute_reply.started":"2025-08-07T02:07:33.185075Z","shell.execute_reply":"2025-08-07T02:07:35.912828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}