{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12557865,"sourceType":"datasetVersion","datasetId":7929546}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 🔴 **Import**","metadata":{}},{"cell_type":"code","source":"import random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set up plot style\nsns.set() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:52:31.508627Z","iopub.execute_input":"2025-07-23T15:52:31.508976Z","iopub.status.idle":"2025-07-23T15:52:32.795034Z","shell.execute_reply.started":"2025-07-23T15:52:31.508951Z","shell.execute_reply":"2025-07-23T15:52:32.794190Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔴 **Args**","metadata":{}},{"cell_type":"code","source":"DIR = \"/kaggle/input/cmi-detect-behavior-with-sensor-data\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:52:38.577893Z","iopub.execute_input":"2025-07-23T15:52:38.578290Z","iopub.status.idle":"2025-07-23T15:52:38.582817Z","shell.execute_reply.started":"2025-07-23T15:52:38.578267Z","shell.execute_reply":"2025-07-23T15:52:38.581773Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔴 **EDA**","metadata":{}},{"cell_type":"markdown","source":"### 🟠 **`train_demographics.csv`**","metadata":{}},{"cell_type":"code","source":"df_dmgc = pd.read_csv(f\"{DIR}/train_demographics.csv\")\ndf_dmgc.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T10:48:49.287786Z","iopub.execute_input":"2025-07-22T10:48:49.288672Z","iopub.status.idle":"2025-07-22T10:48:49.334699Z","shell.execute_reply.started":"2025-07-22T10:48:49.288641Z","shell.execute_reply":"2025-07-22T10:48:49.333809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"🔰 **These tabular files contain demographic and physical characteristics of the participants.**\n\n* **subject**\n* **adult_child**: Indicates whether the participant is a child (0) or an adult (1). Adults are defined as individuals aged 18 years or older.\n* **age**: Participant's age in years at time of data collection.\n* **sex**: Participants sex assigned at birth, 0= female, 1 = male.\n* **handedness**: Dominant hand used by the participant, 0 = left-handed, 1 = right-handed.\n* **height_cm**: Height of the participant in centimeters.\n* **shoulder_to_wrist_cm**: Distance from shoulder to wrist in centimeters.\n* **elbow_to_wrist_cm**: Distance from elbow to wrist in centimeters.\n","metadata":{}},{"cell_type":"code","source":"df_dmgc.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T10:48:52.494358Z","iopub.execute_input":"2025-07-22T10:48:52.494685Z","iopub.status.idle":"2025-07-22T10:48:52.500731Z","shell.execute_reply.started":"2025-07-22T10:48:52.494661Z","shell.execute_reply":"2025-07-22T10:48:52.499932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dmgc.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T10:48:53.328459Z","iopub.execute_input":"2025-07-22T10:48:53.328747Z","iopub.status.idle":"2025-07-22T10:48:53.352206Z","shell.execute_reply.started":"2025-07-22T10:48:53.328725Z","shell.execute_reply":"2025-07-22T10:48:53.351128Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 🟡 **Statistical summary**","metadata":{}},{"cell_type":"code","source":"df_dmgc['adult_child'] = df_dmgc['adult_child'].astype('object')\ndf_dmgc['sex'] = df_dmgc['sex'].astype('object')\ndf_dmgc['handedness'] = df_dmgc['handedness'].astype('object')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:07.841629Z","iopub.execute_input":"2025-07-21T10:46:07.841937Z","iopub.status.idle":"2025-07-21T10:46:07.855185Z","shell.execute_reply.started":"2025-07-21T10:46:07.841907Z","shell.execute_reply":"2025-07-21T10:46:07.854496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dmgc.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:07.855989Z","iopub.execute_input":"2025-07-21T10:46:07.856266Z","iopub.status.idle":"2025-07-21T10:46:07.873784Z","shell.execute_reply.started":"2025-07-21T10:46:07.856239Z","shell.execute_reply":"2025-07-21T10:46:07.873019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dmgc.describe(include='all')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:07.874724Z","iopub.execute_input":"2025-07-21T10:46:07.875059Z","iopub.status.idle":"2025-07-21T10:46:07.914509Z","shell.execute_reply.started":"2025-07-21T10:46:07.875038Z","shell.execute_reply":"2025-07-21T10:46:07.913621Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"🔰 **Summary Insights:**\n\n* Average age is around 21.8 years, with roughly equal numbers of children and adults.\n\n* About 62% of subjects are male, and 88% are right-handed.\n\n* Heights average 168 cm, with a standard deviation of 10.6 cm.\n\n* Arm length measurements also show natural variation consistent with height and age.","metadata":{}},{"cell_type":"code","source":"df_dmgc['adult_child'] = df_dmgc['adult_child'].astype('int64')\ndf_dmgc['sex'] = df_dmgc['sex'].astype('int64')\ndf_dmgc['handedness'] = df_dmgc['handedness'].astype('int64')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:07.915382Z","iopub.execute_input":"2025-07-21T10:46:07.915674Z","iopub.status.idle":"2025-07-21T10:46:07.920692Z","shell.execute_reply.started":"2025-07-21T10:46:07.915647Z","shell.execute_reply":"2025-07-21T10:46:07.919866Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 🟡 **Distribution**","metadata":{}},{"cell_type":"code","source":"# Select only numeric columns\nnumeric_cols = df_dmgc.select_dtypes(include='number').columns\n\n# Set number of rows and columns for subplots\nn_cols = 3\nn_rows = (len(numeric_cols) + n_cols - 1) // n_cols\n\n# Create subplots\nfig, axes = plt.subplots(n_rows, n_cols, figsize=(15, 4 * n_rows))\naxes = axes.flatten()\n\n# Plot histogram for each numeric column\nfor i, col in enumerate(numeric_cols):\n    axes[i].hist(df_dmgc[col], bins=5, color='skyblue', edgecolor='black')\n    axes[i].set_title(f'Histogram of {col}')\n    axes[i].set_xlabel(col)\n    axes[i].set_ylabel('Frequency')\n\n# Remove unused subplots if any\nfor j in range(i + 1, len(axes)):\n    fig.delaxes(axes[j])\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:07.923397Z","iopub.execute_input":"2025-07-21T10:46:07.923634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 🟠 **`train.csv`**","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(f\"{DIR}/train.csv\")\ndf_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:56:21.201300Z","iopub.execute_input":"2025-07-23T17:56:21.201593Z","iopub.status.idle":"2025-07-23T17:56:58.871788Z","shell.execute_reply.started":"2025-07-23T17:56:21.201571Z","shell.execute_reply":"2025-07-23T17:56:58.871016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"🔹 Number of Rows: {df_train.shape[0]} \\n🔹 Number of Cols: {df_train.shape[1]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:42.729674Z","iopub.execute_input":"2025-07-21T10:46:42.729962Z","iopub.status.idle":"2025-07-21T10:46:42.735349Z","shell.execute_reply.started":"2025-07-21T10:46:42.729934Z","shell.execute_reply":"2025-07-21T10:46:42.734524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"🔰 **Columns overview:**\n\n* **row\\_id**: Unique row identifier.\n\n* **sequence\\_id**: An ID for the batch of sensor data. Each sequence includes one **Transition**, one **Pause**, and one **Gesture**.\n\n* **sequence\\_type**: Indicates if the gesture is a **target** or **non-target** type. *(Train only)*\n\n* **sequence\\_counter**: A counter of the row within each sequence.\n\n* **subject**: Unique identifier for the subject who provided the data.\n\n* **gesture**: The target column. Description of the sequence **Gesture**. *(Train only)*\n\n* **orientation**: Description of the subject's orientation during the sequence. *(Train only)*\n\n* **behavior**: Description of the subject's behavior during the current phase of the sequence.\n\n* **acc\\_\\[x/y/z]**: Measures linear acceleration along three axes in meters per second squared from the IMU sensor.\n\n* **rot\\_\\[w/x/y/z]**: Orientation data combining information from the IMU's gyroscope, accelerometer, and magnetometer to describe the device's orientation in 3D space.\n\n* **thm\\_\\[1-5]**: Five thermopile sensors on the watch recording temperature in degrees Celsius. Each index corresponds to the sensor's position (see Overview tab photo).\n\n* **tof\\_\\[1-5]\\_v\\[0-63]**: Five time-of-flight sensors on the watch that measure distance.\n\n  * Columns are named like `tof_1_v0` for the 0th pixel of the first sensor, up to `tof_5_v63` for the 63rd pixel of the fifth sensor.\n  * Data is collected row-wise in an 8×8 grid (top-left to bottom-right).\n  * Each sensor has 64 pixels.\n  * A value of -1 indicates no sensor response (no reflection).\n  * Units are uncalibrated sensor values ranging from 0 to 254.","metadata":{}},{"cell_type":"markdown","source":"#### 🟡 **Statistical Analysis**","metadata":{}},{"cell_type":"code","source":"df_train.describe(include='all')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:42.736267Z","iopub.execute_input":"2025-07-21T10:46:42.736564Z","iopub.status.idle":"2025-07-21T10:46:52.431157Z","shell.execute_reply.started":"2025-07-21T10:46:42.736539Z","shell.execute_reply":"2025-07-21T10:46:52.430145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.behavior.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:52.432117Z","iopub.execute_input":"2025-07-21T10:46:52.432437Z","iopub.status.idle":"2025-07-21T10:46:52.485159Z","shell.execute_reply.started":"2025-07-21T10:46:52.432416Z","shell.execute_reply":"2025-07-21T10:46:52.483839Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### ⚪ **Gesture Class Distribution**","metadata":{}},{"cell_type":"code","source":"# Count the number of unique sequences per gesture\ngesture_counts = df_train[['gesture', 'sequence_id']].drop_duplicates().gesture.value_counts()\n\nplt.figure(figsize=(12, 6))\ngesture_counts.plot(kind='bar')\nplt.title(\"Gesture Class Distribution\")\nplt.xlabel(\"Gesture\")\nplt.ylabel(\"Number of Unique Sequences\")\nplt.xticks(rotation=90)\nplt.grid(True)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:52.486183Z","iopub.execute_input":"2025-07-21T10:46:52.486493Z","iopub.status.idle":"2025-07-21T10:46:53.182497Z","shell.execute_reply.started":"2025-07-21T10:46:52.486471Z","shell.execute_reply":"2025-07-21T10:46:53.181212Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### ⚪ **Number of Sequences per Subject**","metadata":{}},{"cell_type":"code","source":"# Count the number of unique sequences per subject\nsubject_counts = df_train[['subject', 'sequence_id']].drop_duplicates().subject.value_counts()\n\nplt.figure(figsize=(20, 6))\nsubject_counts.sort_index().plot(kind='bar')\nplt.title(\"Number of Sequences per Subject\")\nplt.xlabel(\"Subject\")\nplt.ylabel(\"Number of Unique Sequences\")\nplt.xticks(rotation=90)\nplt.grid(True)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:53.183713Z","iopub.execute_input":"2025-07-21T10:46:53.183959Z","iopub.status.idle":"2025-07-21T10:46:54.676372Z","shell.execute_reply.started":"2025-07-21T10:46:53.183937Z","shell.execute_reply":"2025-07-21T10:46:54.675326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"18*4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:54.677342Z","iopub.execute_input":"2025-07-21T10:46:54.677723Z","iopub.status.idle":"2025-07-21T10:46:54.686446Z","shell.execute_reply.started":"2025-07-21T10:46:54.677674Z","shell.execute_reply":"2025-07-21T10:46:54.685206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose the subject ID you want to analyze\nsubject_id = \"SUBJ_059520\"  # ← Replace with any other subject\n\n# Filter the DataFrame for this subject\ndf_subj = df_train[df_train['subject'] == subject_id]\n\n# Drop duplicates to get one row per sequence\nunique_sequences = df_subj[['gesture', 'orientation', 'sequence_id']].drop_duplicates()\n\n# Count number of unique sequences per (gesture, orientation)\ngesture_orientation_seq_counts = unique_sequences.groupby(['gesture', 'orientation']).size().unstack(fill_value=0)\n\n# Display as table\ngesture_orientation_seq_counts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:54.688291Z","iopub.execute_input":"2025-07-21T10:46:54.688581Z","iopub.status.idle":"2025-07-21T10:46:54.810395Z","shell.execute_reply.started":"2025-07-21T10:46:54.688561Z","shell.execute_reply":"2025-07-21T10:46:54.808791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gesture_orientation_seq_counts.sum()#.sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:54.811736Z","iopub.execute_input":"2025-07-21T10:46:54.811956Z","iopub.status.idle":"2025-07-21T10:46:54.819163Z","shell.execute_reply.started":"2025-07-21T10:46:54.811938Z","shell.execute_reply":"2025-07-21T10:46:54.818373Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### ⚪ **Distribution of Sequence Lengths**","metadata":{}},{"cell_type":"code","source":"# Count the number of rows in each sequence (i.e., sequence length)\nsequence_lengths = df_train.groupby('sequence_id').size()\n\n# Count how many sequences have each unique length\nlength_counts = sequence_lengths.value_counts().sort_index()\n\nplt.figure(figsize=(20, 5))\nlength_counts.plot(kind='bar')\nplt.title(\"Distribution of Sequence Lengths\")\nplt.xlabel(\"Sequence Length (Number of Rows)\")\nplt.ylabel(\"Number of Sequences\")\nplt.grid(True)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:54.820037Z","iopub.execute_input":"2025-07-21T10:46:54.820397Z","iopub.status.idle":"2025-07-21T10:46:57.734978Z","shell.execute_reply.started":"2025-07-21T10:46:54.820370Z","shell.execute_reply":"2025-07-21T10:46:57.734047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"length_counts.sort_values(ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:57.736195Z","iopub.execute_input":"2025-07-21T10:46:57.736742Z","iopub.status.idle":"2025-07-21T10:46:57.744647Z","shell.execute_reply.started":"2025-07-21T10:46:57.736717Z","shell.execute_reply":"2025-07-21T10:46:57.743684Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### ⚪ **Number of Missing Values in Sensor Columns**","metadata":{}},{"cell_type":"code","source":"# Identify sensor columns (acc, rot, thm, tof)\nsensor_cols = [col for col in df_train.columns if (\n    col.startswith('acc_') or col.startswith('rot_') or \n    col.startswith('thm_') or col.startswith('tof_')\n)]\n\n# Count number of NaNs per sensor column\nmissing_values = df_train[sensor_cols].isna().sum()\nmissing_values = missing_values[missing_values > 0].sort_values(ascending=False)\n\nmissing_values[['rot_x', 'thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5', 'tof_1_v0', 'tof_2_v0', 'tof_3_v0', 'tof_4_v0', 'tof_5_v0']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:57.745856Z","iopub.execute_input":"2025-07-21T10:46:57.746350Z","iopub.status.idle":"2025-07-21T10:46:59.105640Z","shell.execute_reply.started":"2025-07-21T10:46:57.746318Z","shell.execute_reply":"2025-07-21T10:46:59.104835Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### ⚪ **Number of -1 Values in TOF Columns**","metadata":{}},{"cell_type":"code","source":"# Extract only TOF columns\ntof_cols = [col for col in df_train.columns if col.startswith('tof_')]\n\n# Count number of -1 values per TOF column\ntof_negative_ones = (df_train[tof_cols] == -1).sum()\n\n# Show only columns with at least one -1\ntof_negative_ones = tof_negative_ones[tof_negative_ones > 0]\n\ntof_negative_ones","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:59.106550Z","iopub.execute_input":"2025-07-21T10:46:59.106950Z","iopub.status.idle":"2025-07-21T10:46:59.845094Z","shell.execute_reply.started":"2025-07-21T10:46:59.106923Z","shell.execute_reply":"2025-07-21T10:46:59.844415Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 🟡 **Sensor Analysis**","metadata":{}},{"cell_type":"code","source":"sequence_id = \"SEQ_000007\"\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\nplt.plot(df_seq[\"acc_x\"])\nplt.plot(df_seq[\"acc_y\"])\nplt.plot(df_seq[\"acc_z\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:46:59.845922Z","iopub.execute_input":"2025-07-21T10:46:59.846201Z","iopub.status.idle":"2025-07-21T10:47:00.123341Z","shell.execute_reply.started":"2025-07-21T10:46:59.846180Z","shell.execute_reply":"2025-07-21T10:47:00.122435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose a specific sequence_id to analyze\nsequence_id = \"SEQ_000007\"\nsequence_id = random.choice(df_train[\"sequence_id\"].unique())\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\n# Plot accelerometer signals over time (sequence_counter)\nplt.figure(figsize=(20, 10))\nplt.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_x\"], label=\"acc_x\")\nplt.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_y\"], label=\"acc_y\")\nplt.plot(df_seq[\"sequence_counter\"], df_seq[\"acc_z\"], label=\"acc_z\")\n\nplt.title(f\"Accelerometer data over time for sequence {sequence_id}\")\nplt.xlabel(\"Sequence Counter\")\nplt.ylabel(\"Acceleration (m/s²)\")\nplt.legend()\nplt.grid(True)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T10:57:22.043634Z","iopub.execute_input":"2025-07-21T10:57:22.044666Z","iopub.status.idle":"2025-07-21T10:57:22.581194Z","shell.execute_reply.started":"2025-07-21T10:57:22.044635Z","shell.execute_reply":"2025-07-21T10:57:22.580252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose a specific sequence_id\nsequence_id = random.choice(df_train[\"sequence_id\"].unique())\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\n# Extract gesture and orientation info\ngesture = df_seq[\"gesture\"].iloc[0]\norientation = df_seq[\"orientation\"].iloc[0]\n\n# Find end index of the 'Transition' phase\ntransition_mask = df_seq[\"phase\"] == \"Transition\"\nif transition_mask.any():\n    transition_end_idx = df_seq[transition_mask][\"sequence_counter\"].max()\nelse:\n    transition_end_idx = None  # Fallback in case 'Transition' phase is missing\n\n# Define accelerometer columns\nacc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n\n# Create subplots\nfig, axs = plt.subplots(len(acc_cols), 1, figsize=(20, 10), sharex=True)\n\nfor i, col in enumerate(acc_cols):\n    axs[i].plot(df_seq[\"sequence_counter\"], df_seq[col], label=col)\n\n    # Mark transition boundary with vertical dashed line\n    if transition_end_idx is not None:\n        axs[i].axvline(x=transition_end_idx, color='red', linestyle='--',\n                       label='Transition → Gesture' if i == 0 else \"\")\n    \n    axs[i].set_ylabel(col)\n    axs[i].legend(loc='upper right')\n\naxs[-1].set_xlabel(\"Sequence Counter\")\n\n# Main title with orientation included\nfig.suptitle(\n    f\"Sensor: Accelerometer | Gesture: {gesture} | Orientation: {orientation} \"\n    f\"{'| Transition → Gesture @ ' + str(transition_end_idx) if transition_end_idx is not None else ''}\",\n    fontsize=16\n)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T11:08:29.979437Z","iopub.execute_input":"2025-07-21T11:08:29.980600Z","iopub.status.idle":"2025-07-21T11:08:30.746425Z","shell.execute_reply.started":"2025-07-21T11:08:29.980565Z","shell.execute_reply":"2025-07-21T11:08:30.745425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose a specific sequence_id\nsequence_id = \"SEQ_000007\"\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\n# Extract gesture and orientation info\ngesture = df_seq[\"gesture\"].iloc[0]\norientation = df_seq[\"orientation\"].iloc[0]\n\n# Define accelerometer columns\nacc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n\n# Create subplots\nfig, axs = plt.subplots(len(acc_cols), 1, figsize=(20, 10), sharex=True)\n\nfor i, col in enumerate(acc_cols):\n    axs[i].plot(df_seq[\"sequence_counter\"], df_seq[col], label=col)\n    \n    # Highlight behavior regions\n    for behavior in df_seq[\"behavior\"].unique():\n        mask = df_seq[\"behavior\"] == behavior\n        xmin = df_seq[mask][\"sequence_counter\"].min()\n        xmax = df_seq[mask][\"sequence_counter\"].max()\n        xmid = (xmin + xmax) / 2\n\n        # Add shaded region\n        axs[i].axvspan(xmin, xmax, color='gray', alpha=0.1)\n\n        # Add behavior label only on top subplot\n        if i == 0:\n            ymax = df_seq[col].max()\n            axs[i].text(xmid, ymax + 0.5, behavior, ha='center', va='bottom', fontsize=10, color='black')\n\n    axs[i].set_ylabel(col)\n    axs[i].legend(loc='upper right')\n\naxs[-1].set_xlabel(\"Sequence Counter\")\n\n# Main title with orientation and gesture\nfig.suptitle(\n    f\"Sensor: Accelerometer | Gesture: {gesture} | Orientation: {orientation}\",\n    fontsize=16\n)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T11:12:47.482334Z","iopub.execute_input":"2025-07-21T11:12:47.482732Z","iopub.status.idle":"2025-07-21T11:12:48.247928Z","shell.execute_reply.started":"2025-07-21T11:12:47.482705Z","shell.execute_reply":"2025-07-21T11:12:48.247042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose a specific sequence_id\nsequence_id = \"SEQ_000007\"\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\n# Extract gesture and orientation info\ngesture = df_seq[\"gesture\"].iloc[0]\norientation = df_seq[\"orientation\"].iloc[0]\n\n# Define rotation columns\nrot_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\n\n# Create subplots for each rotation component\nfig, axs = plt.subplots(len(rot_cols), 1, figsize=(20, 12), sharex=True)\n\nfor i, col in enumerate(rot_cols):\n    axs[i].plot(df_seq[\"sequence_counter\"], df_seq[col], label=col)\n    \n    # Highlight behavior regions with shading and label on top subplot\n    for behavior in df_seq[\"behavior\"].unique():\n        mask = df_seq[\"behavior\"] == behavior\n        xmin = df_seq[mask][\"sequence_counter\"].min()\n        xmax = df_seq[mask][\"sequence_counter\"].max()\n        xmid = (xmin + xmax) / 2\n\n        axs[i].axvspan(xmin, xmax, color='gray', alpha=0.1)\n        \n        if i == 0:\n            ymax = df_seq[col].max()\n            axs[i].text(xmid, ymax + 0.02, behavior, ha='center', va='bottom', fontsize=10, color='black')\n\n    axs[i].set_ylabel(col)\n    axs[i].legend(loc='upper right')\n\naxs[-1].set_xlabel(\"Sequence Counter\")\n\n# Main title with context\nfig.suptitle(\n    f\"Sensor: Rotation | Gesture: {gesture} | Orientation: {orientation}\",\n    fontsize=16\n)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T11:14:44.307295Z","iopub.execute_input":"2025-07-21T11:14:44.307989Z","iopub.status.idle":"2025-07-21T11:14:45.283202Z","shell.execute_reply.started":"2025-07-21T11:14:44.307960Z","shell.execute_reply":"2025-07-21T11:14:45.282296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.spatial.transform import Rotation as R\n\n# Choose a specific sequence_id\nsequence_id = \"SEQ_000007\"\n# sequence_id = random.choice(df_train[\"sequence_id\"].unique())\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\n# Extract gesture and orientation info\ngesture = df_seq[\"gesture\"].iloc[0]\norientation = df_seq[\"orientation\"].iloc[0]\n\n# Convert quaternion to Euler angles (roll, pitch, yaw) in degrees\nquats = df_seq[[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]].to_numpy()\neuler = R.from_quat(quats).as_euler('xyz', degrees=True)  # 'xyz' = roll, pitch, yaw\n\n# Create subplots for Roll, Pitch, Yaw\nfig, axs = plt.subplots(3, 1, figsize=(20, 10), sharex=True)\nlabels = ['Roll (X)', 'Pitch (Y)', 'Yaw (Z)']\n\nfor i in range(3):\n    axs[i].plot(df_seq[\"sequence_counter\"], euler[:, i], label=labels[i], color='tab:blue')\n    \n    # Highlight behavior regions\n    for behavior in df_seq[\"behavior\"].unique():\n        mask = df_seq[\"behavior\"] == behavior\n        xmin = df_seq[mask][\"sequence_counter\"].min()\n        xmax = df_seq[mask][\"sequence_counter\"].max()\n        xmid = (xmin + xmax) / 2\n        axs[i].axvspan(xmin, xmax, color='gray', alpha=0.1)\n        if i == 0:\n            ymax = euler[:, i].max()\n            axs[i].text(xmid, ymax + 7, behavior, ha='center', va='bottom', fontsize=10)\n\n    axs[i].set_ylabel(labels[i])\n    axs[i].legend(loc='upper right')\n\naxs[-1].set_xlabel(\"Sequence Counter\")\n\n# Main title with context\nfig.suptitle(\n    f\"Sensor: Orientation (Euler angles) | Gesture: {gesture} | Orientation: {orientation}\",\n    fontsize=15\n)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T11:22:47.380851Z","iopub.execute_input":"2025-07-21T11:22:47.381457Z","iopub.status.idle":"2025-07-21T11:22:48.153774Z","shell.execute_reply.started":"2025-07-21T11:22:47.381429Z","shell.execute_reply":"2025-07-21T11:22:48.152941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_sequence_signals(df_seq, signal_cols, title_prefix=\"Sensor Signals\"):\n    \"\"\"\n    Plot multiple time-series sensor signals from a single sequence with behavior overlays.\n\n    Parameters:\n    - df_seq: filtered DataFrame for one sequence_id\n    - signal_cols: list of column names (e.g., [\"acc_x\", \"acc_y\", \"acc_z\"])\n    - title_prefix: custom string for title (e.g., \"Rotation\" or \"ToF\")\n    \"\"\"\n    gesture = df_seq[\"gesture\"].iloc[0] if \"gesture\" in df_seq else \"Unknown Gesture\"\n    orientation = df_seq[\"orientation\"].iloc[0] if \"orientation\" in df_seq else \"Unknown Orientation\"\n    \n    fig, axs = plt.subplots(len(signal_cols), 1, figsize=(20, 2.5 * len(signal_cols)), sharex=True)\n\n    # Handle single subplot case\n    if len(signal_cols) == 1:\n        axs = [axs]\n\n    for i, col in enumerate(signal_cols):\n        axs[i].plot(df_seq[\"sequence_counter\"], df_seq[col], label=col, color='tab:blue')\n\n        # Shade behavior regions\n        for behavior in df_seq[\"behavior\"].unique():\n            mask = df_seq[\"behavior\"] == behavior\n            xmin = df_seq[mask][\"sequence_counter\"].min()\n            xmax = df_seq[mask][\"sequence_counter\"].max()\n            xmid = (xmin + xmax) / 2\n            axs[i].axvspan(xmin, xmax, color='gray', alpha=0.1)\n\n            if i == 0:\n                ymax = df_seq[col].max()\n                axs[i].text(xmid, ymax, behavior, ha='center', va='bottom', fontsize=11, color='black')\n\n        axs[i].set_ylabel(col)\n        axs[i].legend(loc='upper right')\n\n    axs[-1].set_xlabel(\"Sequence Counter\")\n\n    fig.suptitle(f\"{title_prefix} | Gesture: {gesture} | Orientation: {orientation}\", fontsize=14)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T11:26:41.994166Z","iopub.execute_input":"2025-07-21T11:26:41.994775Z","iopub.status.idle":"2025-07-21T11:26:42.005886Z","shell.execute_reply.started":"2025-07-21T11:26:41.994744Z","shell.execute_reply":"2025-07-21T11:26:42.004737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose a specific sequence_id\nsequence_id = random.choice(df_train[\"sequence_id\"].unique())\ndf_seq = df_train[df_train[\"sequence_id\"] == sequence_id]\n\n\nplot_sequence_signals(df_seq, [\"acc_x\", \"acc_y\", \"acc_z\"], title_prefix=\"Accelerometer\")\n\nplot_sequence_signals(df_seq, [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"], title_prefix=\"Rotation (Quaternion)\")\n\nplot_sequence_signals(df_seq, [\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"], title_prefix=\"Thermal Sensors\")\n\npixel_index = random.randint(0, 63)\ntof_cols = [f\"tof_{s}_v{pixel_index}\" for s in range(1, 6)]\nplot_sequence_signals(df_seq, tof_cols, title_prefix=\"ToF Sensors (partial)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T11:29:27.960382Z","iopub.execute_input":"2025-07-21T11:29:27.960725Z","iopub.status.idle":"2025-07-21T11:29:31.710264Z","shell.execute_reply.started":"2025-07-21T11:29:27.960701Z","shell.execute_reply":"2025-07-21T11:29:31.709330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"[Quaternions and 3d rotation](https://www.youtube.com/watch?v=zjMuIxRvygQ)","metadata":{}},{"cell_type":"markdown","source":"#### 🟡 **Comparison of Two Subjects with Same Gesture and Orientation**","metadata":{}},{"cell_type":"code","source":"# Randomly select a gesture from available gestures\ngesture = random.choice(df_train[\"gesture\"].dropna().unique())\n\n# Randomly select an orientation from available orientations\norientation = random.choice(df_train[\"orientation\"].dropna().unique())\n\n# Filter the dataframe for rows with the same randomly selected gesture and orientation\ndf_filtered = df_train[(df_train[\"gesture\"] == gesture) & (df_train[\"orientation\"] == orientation)]\n\n# Get the first two distinct subjects that performed this gesture in this orientation\nsubjects = df_filtered[\"subject\"].unique()\n\n# If fewer than two subjects exist for that combination, print a warning and exit\nif len(subjects) < 2:\n    print(f\"⚠️ Not enough subjects found for gesture '{gesture}' and orientation '{orientation}'.\")\nelse:\n    subj1, subj2 = random.sample(list(subjects), 2)  # Pick 2 distinct subjects at random\n\n    # Select one sequence from each subject\n    seq1 = df_filtered[df_filtered[\"subject\"] == subj1][\"sequence_id\"].unique()[0]\n    seq2 = df_filtered[df_filtered[\"subject\"] == subj2][\"sequence_id\"].unique()[0]\n\n    # Filter the dataframe for each selected sequence\n    df_seq1 = df_train[df_train[\"sequence_id\"] == seq1]\n    df_seq2 = df_train[df_train[\"sequence_id\"] == seq2]\n\n    # Choose the signal columns to compare (can be any sensor)\n    signal_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n\n    # Plot both sequences using the same function\n    plot_sequence_signals(df_seq1, signal_cols, title_prefix=f\"Subject: {subj1} | {gesture} | {orientation}\")\n    plot_sequence_signals(df_seq2, signal_cols, title_prefix=f\"Subject: {subj2} | {gesture} | {orientation}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T12:12:55.276970Z","iopub.execute_input":"2025-07-21T12:12:55.277279Z","iopub.status.idle":"2025-07-21T12:12:56.922900Z","shell.execute_reply.started":"2025-07-21T12:12:55.277254Z","shell.execute_reply":"2025-07-21T12:12:56.921924Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 🟡 **Comparison of Two Orientations for the Same Subject and Gesture**","metadata":{}},{"cell_type":"code","source":"# Randomly select a gesture\ngesture = random.choice(df_train[\"gesture\"].dropna().unique())\n\n# Filter rows with that gesture\ndf_gesture = df_train[df_train[\"gesture\"] == gesture]\n\n# Pick a subject that has done this gesture in at least two orientations\nsubject_counts = df_gesture.groupby(\"subject\")[\"orientation\"].nunique()\nvalid_subjects = subject_counts[subject_counts >= 2].index.tolist()\n\nif not valid_subjects:\n    print(f\"⚠️ No subject has performed the gesture '{gesture}' in multiple orientations.\")\nelse:\n    # Select one valid subject randomly\n    subject = random.choice(valid_subjects)\n\n    # Get all orientations this subject performed this gesture in\n    orientations = df_gesture[df_gesture[\"subject\"] == subject][\"orientation\"].unique()\n    orientation1, orientation2 = random.sample(list(orientations), 2)\n\n    # Pick one sequence per orientation\n    seq1 = df_gesture[(df_gesture[\"subject\"] == subject) & (df_gesture[\"orientation\"] == orientation1)][\"sequence_id\"].unique()[0]\n    seq2 = df_gesture[(df_gesture[\"subject\"] == subject) & (df_gesture[\"orientation\"] == orientation2)][\"sequence_id\"].unique()[0]\n\n    # Extract sequences\n    df_seq1 = df_train[df_train[\"sequence_id\"] == seq1]\n    df_seq2 = df_train[df_train[\"sequence_id\"] == seq2]\n\n    # Choose signals to visualize\n    signal_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    signal_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\n\n    # Plot each sequence using the same function\n    plot_sequence_signals(df_seq1, signal_cols, title_prefix=f\"Subject: {subject} | Orientation: {orientation1} | Gesture: {gesture}\")\n    plot_sequence_signals(df_seq2, signal_cols, title_prefix=f\"Subject: {subject} | Orientation: {orientation2} | Gesture: {gesture}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T12:16:19.864153Z","iopub.execute_input":"2025-07-21T12:16:19.864695Z","iopub.status.idle":"2025-07-21T12:16:21.991963Z","shell.execute_reply.started":"2025-07-21T12:16:19.864670Z","shell.execute_reply":"2025-07-21T12:16:21.991211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 🟡 **Visualizing ToF Sensor Grid (8×8) for a Single Time Frame**","metadata":{}},{"cell_type":"code","source":"def plot_tof_grid(df_seq, row_index, sensor_id=1):\n    \"\"\"\n    Plot 8x8 ToF sensor grid for a given row and sensor.\n\n    Parameters:\n    - df_seq: filtered DataFrame for a specific sequence\n    - row_index: index within df_seq (not global df_train)\n    - sensor_id: from 1 to 5\n    \"\"\"\n    # Generate column names for this sensor\n    tof_cols = [f\"tof_{sensor_id}_v{i}\" for i in range(64)]\n    \n    # Convert to float explicitly to avoid type issues\n    values = df_seq.iloc[row_index][tof_cols].astype(float).to_numpy()\n    grid = values.reshape((8, 8)) / 254.\n    \n    # Replace -1 (no reflection) with np.nan to make heatmap clearer\n    grid[grid == -1] = np.nan\n    \n    # Plot heatmap\n    plt.figure(figsize=(6, 5))\n    plt.imshow(grid, cmap='viridis', interpolation='nearest')\n    plt.colorbar(label=\"ToF Value\")\n    plt.title(f\"ToF Sensor {sensor_id} | Frame {row_index} | Sequence: {df_seq['sequence_id'].iloc[0]}\")\n    plt.axis(\"off\")\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T12:18:29.203457Z","iopub.execute_input":"2025-07-21T12:18:29.204320Z","iopub.status.idle":"2025-07-21T12:18:29.210900Z","shell.execute_reply.started":"2025-07-21T12:18:29.204293Z","shell.execute_reply":"2025-07-21T12:18:29.210106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_tof_grid(df_seq, row_index=10, sensor_id=3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-21T12:18:38.090659Z","iopub.execute_input":"2025-07-21T12:18:38.091457Z","iopub.status.idle":"2025-07-21T12:18:38.404764Z","shell.execute_reply.started":"2025-07-21T12:18:38.091429Z","shell.execute_reply":"2025-07-21T12:18:38.403759Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔴 **Preprocessing**","metadata":{}},{"cell_type":"code","source":"df_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:03:33.348001Z","iopub.execute_input":"2025-07-23T12:03:33.348723Z","iopub.status.idle":"2025-07-23T12:03:33.593776Z","shell.execute_reply.started":"2025-07-23T12:03:33.348659Z","shell.execute_reply":"2025-07-23T12:03:33.592984Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.iloc[:, 8:].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:06:00.614996Z","iopub.execute_input":"2025-07-23T12:06:00.615870Z","iopub.status.idle":"2025-07-23T12:06:02.504832Z","shell.execute_reply.started":"2025-07-23T12:06:00.615838Z","shell.execute_reply":"2025-07-23T12:06:02.504011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:07:02.795802Z","iopub.execute_input":"2025-07-23T12:07:02.796137Z","iopub.status.idle":"2025-07-23T12:07:11.520444Z","shell.execute_reply.started":"2025-07-23T12:07:02.796109Z","shell.execute_reply":"2025-07-23T12:07:11.519482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[df_train['tof_5_v1'] > -1].tof_5_v1.median()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T13:25:22.587634Z","iopub.execute_input":"2025-07-23T13:25:22.588054Z","iopub.status.idle":"2025-07-23T13:25:23.005041Z","shell.execute_reply.started":"2025-07-23T13:25:22.588027Z","shell.execute_reply":"2025-07-23T13:25:23.004221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define feature groups\nacc_cols = [col for col in df_train.columns if col.startswith(\"acc_\")]\nrot_cols = [col for col in df_train.columns if col.startswith(\"rot_\")]\nthm_cols = [col for col in df_train.columns if col.startswith(\"thm_\")]\ntof_cols = [f\"tof_{i}_v{0}\" for i in range(1, 6)]\n\n\ndef plot_combined_box_violin(df, columns, title):\n    data = [df[col].dropna().values for col in columns]\n\n    fig, ax = plt.subplots(figsize=(20, 10))\n\n    # Horizontal boxplot on top of violin plot\n    ax.boxplot(data, notch=False, patch_artist=True, widths=0.1, vert=False)\n\n    # Horizontal violin plot\n    ax.violinplot(data, showmeans=False, showmedians=False, showextrema=False, vert=False)\n\n    # Axis settings\n    ax.set_title(title)\n    ax.set_yticks(range(1, len(columns) + 1))\n    ax.set_yticklabels(columns)\n    ax.grid(True, axis='x')  # Grid on x-axis since horizontal plot\n\n    plt.tight_layout()\n    plt.show()\n\n\n# Run plots\nplot_combined_box_violin(df_train, acc_cols, \"Accelerometer Sensors\")\nplot_combined_box_violin(df_train, rot_cols, \"Rotation Sensors\")\nplot_combined_box_violin(df_train, thm_cols, \"Thermal Sensors\")\nplot_combined_box_violin(df_train, tof_cols, \"ToF Sensors (first pixel per group)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:12:00.913526Z","iopub.execute_input":"2025-07-23T12:12:00.913925Z","iopub.status.idle":"2025-07-23T12:12:42.452385Z","shell.execute_reply.started":"2025-07-23T12:12:00.913896Z","shell.execute_reply":"2025-07-23T12:12:42.451331Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 🟠 **Outliers**","metadata":{}},{"cell_type":"code","source":"# Find outliers in thm_1\noutliers_thm1 = df_train[df_train['thm_1'] < 10]\ndisplay(outliers_thm1[thm_cols])\n\n# Find outliers in thm_3\noutliers_thm3 = df_train[df_train['thm_3'] == 0]\ndisplay(outliers_thm3[thm_cols])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:55:55.931741Z","iopub.execute_input":"2025-07-23T12:55:55.932108Z","iopub.status.idle":"2025-07-23T12:55:55.979202Z","shell.execute_reply.started":"2025-07-23T12:55:55.932082Z","shell.execute_reply":"2025-07-23T12:55:55.978331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thm_target = 'thm_3'\n\n# Identify all sequence_ids where thm_3 is zero in at least one row\nsequences_with_zero_thm3 = df_train[df_train[thm_target] == 0]['sequence_id'].unique()\n\n# Create a dataframe only for those sequences\nsubset = df_train[df_train['sequence_id'].isin(sequences_with_zero_thm3)]\n\n# For each sequence, check if *all* thm_3 values are zero or just some\nsequence_zero_stats = subset.groupby('sequence_id')[thm_target].agg(\n    total_frames='count',\n    zero_count=lambda x: (x == 0).sum()\n)\n\n# Step 4: Add a column showing if the entire sequence is zero or partial\nsequence_zero_stats['zero_type'] = sequence_zero_stats.apply(\n    lambda row: 'All Zero' if row['total_frames'] == row['zero_count'] else 'Partial Zero',\n    axis=1\n)\n\n# Display results\nprint(sequence_zero_stats['zero_type'].value_counts())  # Count how many are \"All Zero\" vs \"Partial Zero\"\ndisplay(sequence_zero_stats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:57:17.484088Z","iopub.execute_input":"2025-07-23T12:57:17.485114Z","iopub.status.idle":"2025-07-23T12:57:17.558207Z","shell.execute_reply.started":"2025-07-23T12:57:17.485073Z","shell.execute_reply":"2025-07-23T12:57:17.557363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thm_target = 'thm_1'\n\n# Identify all sequence_ids where thm_3 is zero in at least one row\nsequences_with_zero_thm3 = df_train[df_train[thm_target] < 10]['sequence_id'].unique()\n\n# Create a dataframe only for those sequences\nsubset = df_train[df_train['sequence_id'].isin(sequences_with_zero_thm3)]\n\n# For each sequence, check if *all* thm_3 values are zero or just some\nsequence_zero_stats = subset.groupby('sequence_id')[thm_target].agg(\n    total_frames='count',\n    zero_count=lambda x: (x < 10).sum()\n)\n\n# Step 4: Add a column showing if the entire sequence is zero or partial\nsequence_zero_stats['zero_type'] = sequence_zero_stats.apply(\n    lambda row: 'All Zero' if row['total_frames'] == row['zero_count'] else 'Partial Zero',\n    axis=1\n)\n\n# Display results\nprint(sequence_zero_stats['zero_type'].value_counts())  # Count how many are \"All Zero\" vs \"Partial Zero\"\ndisplay(sequence_zero_stats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T12:59:01.615961Z","iopub.execute_input":"2025-07-23T12:59:01.616290Z","iopub.status.idle":"2025-07-23T12:59:01.680427Z","shell.execute_reply.started":"2025-07-23T12:59:01.616270Z","shell.execute_reply":"2025-07-23T12:59:01.679470Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def impute_low_thermal_values(df, thm_columns, threshold=10.0):\n    # Create a mask of low values\n    low_mask = df[thm_columns] < threshold\n\n    # For each row, replace low values with the mean of other valid (>= threshold) thermal values\n    for col in thm_columns:\n        low_rows = low_mask[col]\n\n        # Compute row-wise means excluding the low value itself\n        row_means = df.loc[low_rows, thm_columns].mask(low_mask, np.nan).mean(axis=1)\n\n        # Impute\n        df.loc[low_rows, col] = row_means\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:53:27.462282Z","iopub.execute_input":"2025-07-23T15:53:27.462642Z","iopub.status.idle":"2025-07-23T15:53:27.468065Z","shell.execute_reply.started":"2025-07-23T15:53:27.462618Z","shell.execute_reply":"2025-07-23T15:53:27.466949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = impute_low_thermal_values(df_train, thm_cols, threshold=10.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:54:10.902553Z","iopub.execute_input":"2025-07-23T15:54:10.903204Z","iopub.status.idle":"2025-07-23T15:54:11.048998Z","shell.execute_reply.started":"2025-07-23T15:54:10.903176Z","shell.execute_reply":"2025-07-23T15:54:11.048161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_combined_box_violin(df_train, thm_cols, \"Thermal Sensors\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T15:54:42.239923Z","iopub.execute_input":"2025-07-23T15:54:42.240220Z","iopub.status.idle":"2025-07-23T15:54:55.450886Z","shell.execute_reply.started":"2025-07-23T15:54:42.240197Z","shell.execute_reply":"2025-07-23T15:54:55.450010Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 🟠 **Missing Values**","metadata":{}},{"cell_type":"code","source":"df_train.iloc[:, 9:].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T16:12:23.779588Z","iopub.execute_input":"2025-07-23T16:12:23.779959Z","iopub.status.idle":"2025-07-23T16:12:24.872883Z","shell.execute_reply.started":"2025-07-23T16:12:23.779934Z","shell.execute_reply":"2025-07-23T16:12:24.871993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Total number of rows in the dataset\ntotal_rows = len(df_train)\n\n# Define sensor groups (using one column per ToF sensor)\nsensor_groups = {\n    \"Accelerometer\": [\"acc_x\", \"acc_y\", \"acc_z\"],\n    \"Rotation\": [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    \"Thermal thm_1\": [\"thm_1\"],\n    \"Thermal thm_2\": [\"thm_2\"],\n    \"Thermal thm_3\": [\"thm_3\"],\n    \"Thermal thm_4\": [\"thm_4\"],\n    \"Thermal thm_5\": [\"thm_5\"],\n    \"ToF 1 (64 cols)\": [\"tof_1_v0\"],\n    \"ToF 2 (64 cols)\": [\"tof_2_v0\"],\n    \"ToF 3 (64 cols)\": [\"tof_3_v0\"],\n    \"ToF 4 (64 cols)\": [\"tof_4_v0\"],\n    \"ToF 5 (64 cols)\": [\"tof_5_v0\"],\n}\n\n# Initialize list to collect results\nmissing_data = []\n\n# Loop over each sensor group and calculate missing values\nfor sensor_name, columns in sensor_groups.items():\n    # Count missing values from the representative column(s)\n    total_missing = df_train[columns].isna().sum().sum()\n    # Compute missing percentage\n    missing_percentage = (total_missing / total_rows) * 100\n    # Append result\n    missing_data.append({\n        \"Sensor Type\": sensor_name,\n        \"Missing Count\": int(total_missing),\n        \"Missing Percentage\": f\"{missing_percentage:.2f}%\"\n    })\n\n# Create summary DataFrame\ndf_missing_summary = pd.DataFrame(missing_data)\n\n# Display result\ndf_missing_summary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T16:13:16.602059Z","iopub.execute_input":"2025-07-23T16:13:16.602375Z","iopub.status.idle":"2025-07-23T16:13:16.672610Z","shell.execute_reply.started":"2025-07-23T16:13:16.602354Z","shell.execute_reply":"2025-07-23T16:13:16.671793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sensor groups\nacc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\nrot_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\ntof_sensors = [f\"tof_{i}_\" for i in range(1, 6)]\nthm_sensors = [f\"thm_{i}\" for i in range(1, 6)]\n\n# Result storage\nsensor_results = []\n\n\n# Function to analyze missingness for a sensor group\ndef analyze_sensor_missingness(df, feature_cols, sensor_name):\n    nan_stats = (\n        df.groupby(\"sequence_id\")[feature_cols]\n        .apply(lambda x: x.isna().mean())\n        .reset_index()\n    )\n\n    nan_stats[\"max_nan_ratio\"] = nan_stats[feature_cols].max(axis=1)\n    nan_stats[\"min_nan_ratio\"] = nan_stats[feature_cols].min(axis=1)\n\n    sequences_with_nan = nan_stats[nan_stats[\"max_nan_ratio\"] > 0]\n    fully_missing = nan_stats[nan_stats[\"max_nan_ratio\"] == 1.0]\n    partially_missing = nan_stats[\n        (nan_stats[\"max_nan_ratio\"] > 0) & (nan_stats[\"max_nan_ratio\"] < 1.0)\n    ]\n    nan_stats[\"nan_overlap\"] = nan_stats[\"max_nan_ratio\"] == nan_stats[\"min_nan_ratio\"]\n    overlapping = nan_stats[\n        (nan_stats[\"max_nan_ratio\"] > 0) & (nan_stats[\"nan_overlap\"] == True)\n    ]\n\n    sensor_results.append({\n        \"Sensor\": sensor_name,\n        \"Any Missing\": len(sequences_with_nan),\n        \"Fully Missing\": len(fully_missing),\n        \"Partially Missing\": len(partially_missing),\n        \"Overlap Missing\": len(overlapping),\n        \"Independent Missing\": len(sequences_with_nan) - len(overlapping)\n    })\n\n\n# Analyze accelerometer\nanalyze_sensor_missingness(df_train, acc_cols, \"acc\")\n\n# Analyze rotation\nanalyze_sensor_missingness(df_train, rot_cols, \"rot\")\n\n# Analyze thermal sensors\nfor thm in thm_sensors:\n    analyze_sensor_missingness(df_train, [thm], thm)\n\n# Analyze ToF sensors\nfor i, base in enumerate(tof_sensors, 1):\n    tof_cols = [f\"{base}v{j}\" for j in range(64)]\n    analyze_sensor_missingness(df_train, tof_cols, f\"tof_{i}\")\n\n# Create and display result table\ndf_sensor_missing = pd.DataFrame(sensor_results)\ndf_sensor_missing","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T16:21:37.668850Z","iopub.execute_input":"2025-07-23T16:21:37.669862Z","iopub.status.idle":"2025-07-23T16:22:07.771070Z","shell.execute_reply.started":"2025-07-23T16:21:37.669832Z","shell.execute_reply":"2025-07-23T16:22:07.770066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define sensor groups\nacc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\nrot_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\nthm_sensors = [f\"thm_{i}\" for i in range(1, 6)]\ntof_sensors = [f\"tof_{i}_v{j}\" for i in range(1, 6) for j in range(64)]\n\n# Organize sensor groups into a dictionary\nsensor_groups = {\n    \"acc\": acc_cols,\n    \"rot\": rot_cols,\n    \"thm\": thm_sensors,\n    \"tof\": tof_sensors\n}\n\n# Compute NaN ratio per column for each sequence\nnan_ratio_per_seq = df_train.groupby(\"sequence_id\").apply(lambda x: x.isna().mean())\n\n# For each sequence, count how many columns (sensors) in each group have missing values\nsequence_missing_counts = pd.DataFrame(index=nan_ratio_per_seq.index)\n\nfor group_name, group_cols in sensor_groups.items():\n    group_nan = nan_ratio_per_seq[group_cols]\n    # Count how many columns in the group have any NaNs (ratio > 0)\n    sequence_missing_counts[f\"{group_name}_missing_sensors\"] = (group_nan > 0).sum(axis=1)\n\n# Summarize: For each group, how many sequences have N sensors missing (N = 1, 2, ..., 320)\nsummary = {}\nfor group in [\"thm\", \"tof\"]:\n    counts = sequence_missing_counts[f\"{group}_missing_sensors\"].value_counts().sort_index()\n    summary[group] = counts\n\n# Convert summary into a DataFrame\nsummary_df = pd.DataFrame(summary).fillna(0).astype(int)\nsummary_df.index.name = \"Num sensors with NaN\"\nsummary_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T16:58:08.010751Z","iopub.execute_input":"2025-07-23T16:58:08.011132Z","iopub.status.idle":"2025-07-23T16:58:15.211970Z","shell.execute_reply.started":"2025-07-23T16:58:08.011104Z","shell.execute_reply":"2025-07-23T16:58:15.211125Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Copy the sequence-level missing sensor counts\ncombo_df = sequence_missing_counts.copy()\n\n# Convert counts to boolean: True if any sensor in the group is missing\ncombo_df_binary = combo_df > 0\n\n# Analyze sensor group combinations with missing values\ncombo_df_binary[\"thm_and_rot\"] = combo_df_binary[\"thm_missing_sensors\"] & combo_df_binary[\"rot_missing_sensors\"]\ncombo_df_binary[\"thm_and_tof\"] = combo_df_binary[\"thm_missing_sensors\"] & combo_df_binary[\"tof_missing_sensors\"]\ncombo_df_binary[\"rot_and_tof\"] = combo_df_binary[\"rot_missing_sensors\"] & combo_df_binary[\"tof_missing_sensors\"]\n\n# All three sensor groups missing\ncombo_df_binary[\"all_three\"] = (\n    combo_df_binary[\"thm_missing_sensors\"]\n    & combo_df_binary[\"rot_missing_sensors\"]\n    & combo_df_binary[\"tof_missing_sensors\"]\n)\n\n# Count how many sequences fall into each combination\ncombo_stats = combo_df_binary[[\"thm_and_rot\", \"thm_and_tof\", \"rot_and_tof\", \"all_three\"]].sum()\ncombo_stats\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:02:13.217153Z","iopub.execute_input":"2025-07-23T17:02:13.217502Z","iopub.status.idle":"2025-07-23T17:02:13.231561Z","shell.execute_reply.started":"2025-07-23T17:02:13.217475Z","shell.execute_reply":"2025-07-23T17:02:13.230790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get all ToF columns (those that start with 'tof_')\ntof_columns = [col for col in df_train.columns if col.startswith(\"tof_\")]\n\n# Total number of rows\ntotal_rows = len(df_train)\n\n# Create list to store results\ntof_neg1_stats = []\n\n# Loop through each ToF column and count -1 values\nfor col in tof_columns:\n    count_neg1 = (df_train[col] == -1).sum()\n    percent_neg1 = (count_neg1 / total_rows) * 100\n    tof_neg1_stats.append({\n        \"ToF Column\": col,\n        \"Count of -1\": int(count_neg1),\n        \"Percentage of -1\": f\"{percent_neg1:.2f}%\"\n    })\n\n# Convert list to DataFrame\ndf_neg1_summary = pd.DataFrame(tof_neg1_stats)\n\n# Display summary\ndf_neg1_summary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:04:00.484510Z","iopub.execute_input":"2025-07-23T17:04:00.485356Z","iopub.status.idle":"2025-07-23T17:04:01.003498Z","shell.execute_reply.started":"2025-07-23T17:04:00.485324Z","shell.execute_reply":"2025-07-23T17:04:01.002639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_seq.ffill().bfill().fillna(0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:28:13.166934Z","iopub.execute_input":"2025-07-23T17:28:13.167271Z","iopub.status.idle":"2025-07-23T17:28:21.488459Z","shell.execute_reply.started":"2025-07-23T17:28:13.167245Z","shell.execute_reply":"2025-07-23T17:28:21.487536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\n\n\nclass SensorImputer(BaseEstimator, TransformerMixin):\n    def __init__(self, method='mean'):\n        self.method = method  # 'mean' or 'median'\n        self.acc_columns = []\n        self.rot_columns = []\n        self.thm_columns = []\n        self.tof_columns = []\n        self.column_stats = {}\n\n    def fit(self, X, y=None):\n        # Identify sensor columns\n        self.acc_columns = [col for col in X.columns if col.startswith(\"acc_\")]\n        self.rot_columns = [col for col in X.columns if col.startswith(\"rot_\")]\n        self.thm_columns = [col for col in X.columns if col.startswith(\"thm_\")]\n        self.tof_columns = [col for col in X.columns if col.startswith(\"tof_\")]\n\n        # Calculate statistics for acc, rot, and thm columns\n        sensor_cols = self.acc_columns + self.rot_columns + self.thm_columns\n        if self.method == 'mean':\n            self.column_stats.update({col: X[col].mean() for col in sensor_cols})\n        elif self.method == 'median':\n            self.column_stats.update({col: X[col].median() for col in sensor_cols})\n        else:\n            raise ValueError(\"Method must be either 'mean' or 'median'\")\n\n        # Calculate statistics for tof columns (ignoring -1 values)\n        for col in self.tof_columns:\n            valid_values = X[col][X[col] != -1]\n            if self.method == 'mean':\n                self.column_stats[col] = valid_values.mean()\n            elif self.method == 'median':\n                self.column_stats[col] = valid_values.median()\n\n        return self\n\n    def transform(self, X):\n        X = X.copy()\n\n        # Fill missing values in acc, rot, and thm columns using precomputed stats\n        for col in self.acc_columns + self.rot_columns + self.thm_columns:\n            X[col] = X[col].fillna(self.column_stats[col])\n\n        # Replace -1 with NaN in tof columns, then fill with precomputed stats\n        for col in self.tof_columns:\n            X[col] = X[col].replace(-1, np.nan)\n            X[col] = X[col].fillna(self.column_stats[col])  # 255\n\n        return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:38:16.985005Z","iopub.execute_input":"2025-07-23T17:38:16.985455Z","iopub.status.idle":"2025-07-23T17:38:17.107308Z","shell.execute_reply.started":"2025-07-23T17:38:16.985426Z","shell.execute_reply":"2025-07-23T17:38:17.106459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = SensorImputer(method='median')\nimputer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:38:27.805017Z","iopub.execute_input":"2025-07-23T17:38:27.805866Z","iopub.status.idle":"2025-07-23T17:38:27.818318Z","shell.execute_reply.started":"2025-07-23T17:38:27.805839Z","shell.execute_reply":"2025-07-23T17:38:27.817625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_imputed = imputer.fit_transform(df_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:39:25.709493Z","iopub.execute_input":"2025-07-23T17:39:25.709931Z","iopub.status.idle":"2025-07-23T17:39:34.446262Z","shell.execute_reply.started":"2025-07-23T17:39:25.709904Z","shell.execute_reply":"2025-07-23T17:39:34.445376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_imputed.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:40:05.895205Z","iopub.execute_input":"2025-07-23T17:40:05.895499Z","iopub.status.idle":"2025-07-23T17:40:06.940100Z","shell.execute_reply.started":"2025-07-23T17:40:05.895475Z","shell.execute_reply":"2025-07-23T17:40:06.939239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_combined_box_violin(df_train, acc_cols, \"Accelerometer Sensors\")\n\nplot_combined_box_violin(df_train, rot_cols, \"Rotation Sensors\")\n\nplot_combined_box_violin(df_train, thm_cols, \"Thermal Sensors\")\n\ntof_cols = [f\"tof_{i}_v{0}\" for i in range(1, 6)]\nplot_combined_box_violin(df_train, tof_cols, \"ToF Sensors (first pixel per group)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:46:04.407617Z","iopub.execute_input":"2025-07-23T17:46:04.408014Z","iopub.status.idle":"2025-07-23T17:46:43.234245Z","shell.execute_reply.started":"2025-07-23T17:46:04.407988Z","shell.execute_reply":"2025-07-23T17:46:43.233308Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 🟠 **Normalization**","metadata":{}},{"cell_type":"code","source":"df_train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:57:25.587109Z","iopub.execute_input":"2025-07-23T17:57:25.587433Z","iopub.status.idle":"2025-07-23T17:57:33.672703Z","shell.execute_reply.started":"2025-07-23T17:57:25.587409Z","shell.execute_reply":"2025-07-23T17:57:33.671898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Select sensor columns (excluding rotation)\nfeature_cols = [col for col in df_train.columns if col.startswith((\"acc_\", \"thm_\", \"tof_\"))]\n\n# Initialize scaler\nscaler = StandardScaler()\n\n# Apply Z-score normalization\ndf_train[feature_cols] = scaler.fit_transform(df_train[feature_cols])\n\ndf_train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T18:07:51.771590Z","iopub.execute_input":"2025-07-23T18:07:51.772191Z","iopub.status.idle":"2025-07-23T18:08:10.198005Z","shell.execute_reply.started":"2025-07-23T18:07:51.772155Z","shell.execute_reply":"2025-07-23T18:08:10.197159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_imputed.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T17:55:27.626225Z","iopub.execute_input":"2025-07-23T17:55:27.627047Z","iopub.status.idle":"2025-07-23T17:55:36.204792Z","shell.execute_reply.started":"2025-07-23T17:55:27.627018Z","shell.execute_reply":"2025-07-23T17:55:36.203962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.preprocessing import StandardScaler\n\n# Select sensor columns (excluding rotation)\nfeature_cols = [col for col in df_train.columns if col.startswith((\"acc_\", \"thm_\", \"tof_\"))]\n\n# Initialize scaler\nscaler = StandardScaler()\n\n# Apply Z-score normalization\ndf_train_imputed[feature_cols] = scaler.fit_transform(df_train_imputed[feature_cols])\n\ndf_train_imputed.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T18:11:36.216354Z","iopub.execute_input":"2025-07-23T18:11:36.216704Z","iopub.status.idle":"2025-07-23T18:11:49.667183Z","shell.execute_reply.started":"2025-07-23T18:11:36.216664Z","shell.execute_reply":"2025-07-23T18:11:49.666199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_imputed.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T18:13:44.336412Z","iopub.execute_input":"2025-07-23T18:13:44.336889Z","iopub.status.idle":"2025-07-23T18:13:44.363881Z","shell.execute_reply.started":"2025-07-23T18:13:44.336859Z","shell.execute_reply":"2025-07-23T18:13:44.363055Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 🟠 **Label Encoding**","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\ndf_train_imputed['gesture'] = le.fit_transform(df_train_imputed['gesture'])\ndf_train_imputed.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T18:18:54.527994Z","iopub.execute_input":"2025-07-23T18:18:54.528360Z","iopub.status.idle":"2025-07-23T18:18:54.663669Z","shell.execute_reply.started":"2025-07-23T18:18:54.528332Z","shell.execute_reply":"2025-07-23T18:18:54.662963Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔴 **Save Preprocessed Dataset**","metadata":{}},{"cell_type":"code","source":"# df_train_imputed.to_csv(\"train-preprocessed.csv\", index=False)\ndf_train_imputed.sample(100).to_csv(\"train-preprocessed.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T18:38:38.863797Z","iopub.execute_input":"2025-07-23T18:38:38.864573Z","iopub.status.idle":"2025-07-23T18:38:38.974442Z","shell.execute_reply.started":"2025-07-23T18:38:38.864543Z","shell.execute_reply":"2025-07-23T18:38:38.973603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# /kaggle/input/cmi25-train-preprocessed/train-preprocessed.csv","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}