{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import section","metadata":{}},{"cell_type":"markdown","source":"This section is where all the module imports are done.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport math\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.metrics import f1_score, accuracy_score\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport random\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\n\n# Set random seed for reproducibility\ntorch.manual_seed(42)\nnp.random.seed(42)\nrandom.seed(42)","metadata":{"_uuid":"33af8552-c01f-4d0e-94b7-3936ea90751b","_cell_guid":"22a417d4-c0ee-4d38-9330-188dedf7f897","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:08:41.870732Z","iopub.execute_input":"2025-08-20T14:08:41.871000Z","iopub.status.idle":"2025-08-20T14:08:47.661612Z","shell.execute_reply.started":"2025-08-20T14:08:41.870979Z","shell.execute_reply":"2025-08-20T14:08:47.660820Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{"_uuid":"2d564ce2-71ca-4d6f-9581-34024bd09e24","_cell_guid":"a1f163cf-fcd3-4657-9f64-848ae6d4b1a0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"In this section, we explore the raw training and test data from different sensors (IMU, TOF, and thermopiles), along with demographic information. The goal is to get an initial sense of the distributions, spot anomalies, and identify potential issues such as missing or faulty sensor readings.","metadata":{}},{"cell_type":"code","source":"# Paths to data\ntrain_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\ntest_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\"\ndemographic_train_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\"\ndemographic_test_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\"\n\n# Read data\ntrain_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)\ndemographic_train_df = pd.read_csv(demographic_train_path)\ndemographic_test_df = pd.read_csv(demographic_test_path)","metadata":{"_uuid":"7cc1eee9-5b65-4924-870e-df644fc7726f","_cell_guid":"0cd52c90-61f7-4463-bb8e-9aa9a7909fc2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:08:47.662895Z","iopub.execute_input":"2025-08-20T14:08:47.663299Z","iopub.status.idle":"2025-08-20T14:09:17.578444Z","shell.execute_reply.started":"2025-08-20T14:08:47.663271Z","shell.execute_reply":"2025-08-20T14:09:17.577822Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# join both datasets\ntrain_df = train_df.merge(demographic_train_df, on=\"subject\", how=\"left\")\ntest_df = test_df.merge(demographic_test_df, on=\"subject\", how=\"left\")","metadata":{"_uuid":"6232f7b8-86ab-4efb-94f5-a9b3e7ccb6f8","_cell_guid":"e3afc609-5912-4c74-a4fa-5686d45f5052","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:17.579140Z","iopub.execute_input":"2025-08-20T14:09:17.579368Z","iopub.status.idle":"2025-08-20T14:09:18.216119Z","shell.execute_reply.started":"2025-08-20T14:09:17.579352Z","shell.execute_reply":"2025-08-20T14:09:18.215386Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## IMU Analysis","metadata":{"_uuid":"ab826165-8eb7-46ec-b4fe-87a95863dbde","_cell_guid":"315647ec-f6d4-4fc8-abcc-3283f01448a7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"We take a closer look at the accelerometer and rotation data. This helps establish the typical ranges of motion and verify whether the signals behave as expected before modelling.","metadata":{}},{"cell_type":"code","source":"train_df.filter(items=['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']).describe()","metadata":{"_uuid":"f51a573a-426a-4f0f-9905-6101f943dead","_cell_guid":"6cc4e8c2-a86e-40f8-a28d-44f17734e9f8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:18.217901Z","iopub.execute_input":"2025-08-20T14:09:18.218117Z","iopub.status.idle":"2025-08-20T14:09:18.382088Z","shell.execute_reply.started":"2025-08-20T14:09:18.218100Z","shell.execute_reply":"2025-08-20T14:09:18.381389Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TOF Analysis","metadata":{"_uuid":"67491a9d-64da-457d-8422-bde5723f9588","_cell_guid":"39bf3795-3438-4d07-a64d-e753b019b992","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Here we investigate the time-of-flight sensors. The focus is on understanding the spread and consistency of the distance measurements, while checking for unusual values.","metadata":{}},{"cell_type":"code","source":"train_df.filter(like='tof_').describe()","metadata":{"_uuid":"aa8bd458-988e-4e6e-8fe0-3bd4e1fbd531","_cell_guid":"57913725-af83-4577-8fd3-761321da492f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:18.383161Z","iopub.execute_input":"2025-08-20T14:09:18.383427Z","iopub.status.idle":"2025-08-20T14:09:25.374234Z","shell.execute_reply.started":"2025-08-20T14:09:18.383404Z","shell.execute_reply":"2025-08-20T14:09:25.373653Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## THM Analysis","metadata":{"_uuid":"ed16dfe5-996d-4a79-b908-872f34202ac8","_cell_guid":"776cc759-c5bc-4fb9-803c-bbdb546bb7d7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Thermopile sensor readings are examined in detail. We highlight unrealistic values (e.g., negative or near-zero temperatures) that suggest sensor malfunctions and outline a plan to address these.","metadata":{}},{"cell_type":"code","source":"train_df.filter(like='thm_').describe()","metadata":{"_uuid":"aa762dac-8192-4358-bbc3-d13fcb0e11b8","_cell_guid":"9d049780-eb7b-4a9e-8780-6c3c58359a99","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:25.374926Z","iopub.execute_input":"2025-08-20T14:09:25.375169Z","iopub.status.idle":"2025-08-20T14:09:25.492470Z","shell.execute_reply.started":"2025-08-20T14:09:25.375143Z","shell.execute_reply":"2025-08-20T14:09:25.491720Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Via the describe() method, we can see there are some weird values for some of the thermopiles, namely 1 and 3, where the minimum values range from negative values and 0 degrees. These values do not make much sense, since they read the temperature in Celcius and are very close to each other, so any big difference would mean a malfunction from the sensor.\n\nWe will analyse these malfunctions below.","metadata":{"_uuid":"1591fa31-7d83-488d-886e-a0c342e1c247","_cell_guid":"3942f2c4-7268-412c-9579-b7e8bc0b432a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Create a mask for rows where any 'thm_' column is below the threshold\nmask = (train_df.filter(like='thm_') < 10).any(axis=1)\nfiltered_df = train_df[mask]\n\n# Get all unique sequence IDs with values below the threshold\nseqs_with_low_values = list(filtered_df['sequence_id'].unique())\n\nprint(f\"There are {len(seqs_with_low_values)} sequences with erroneous thermopile values.\")","metadata":{"_uuid":"9c9bd5f5-933f-4eaa-84fc-d15e1b782788","_cell_guid":"0a3c3ace-8b5a-4db6-8210-2960b1cfc60f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:25.493355Z","iopub.execute_input":"2025-08-20T14:09:25.493620Z","iopub.status.idle":"2025-08-20T14:09:25.521724Z","shell.execute_reply.started":"2025-08-20T14:09:25.493597Z","shell.execute_reply":"2025-08-20T14:09:25.521077Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select 4 random sequence_ids\nsampled_seqs = random.sample(seqs_with_low_values, 4)\n\nfig, axes = plt.subplots(2, 2, figsize=(10, 4))\naxes = axes.flatten()\n\nfor idx, seq_id in enumerate(sampled_seqs):\n    ax = axes[idx]\n    seq_df = train_df[train_df['sequence_id'] == seq_id]\n    \n    for i in range(1, 6):\n        ax.plot(seq_df['sequence_counter'], seq_df[f'thm_{i}'], label=f'thm_{i}', linewidth=0.8)\n    \n    ax.set_title(f\"Seq {seq_id}\")\n    ax.set_xlabel(\"sequence_counter\")\n    ax.set_ylabel(\"thm values\")\n    ax.grid(True)\n\n# Hide any unused axes (if fewer than 4)\nfor j in range(len(sampled_seqs), 4):\n    axes[j].axis('off')\n\nfig.tight_layout()\nplt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=5)\nplt.show()","metadata":{"_uuid":"0c80a982-39af-46da-9d73-928d0f0efae2","_cell_guid":"8538f33f-5e47-4b3b-8fc5-fa6d2b78970e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:25.522401Z","iopub.execute_input":"2025-08-20T14:09:25.522587Z","iopub.status.idle":"2025-08-20T14:09:26.316637Z","shell.execute_reply.started":"2025-08-20T14:09:25.522571Z","shell.execute_reply":"2025-08-20T14:09:26.315840Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1D-CNN - Transformer: Data Preprocessing","metadata":{"_uuid":"5a28572a-d60b-4334-a72b-886daa5bb42b","_cell_guid":"d05eb376-720a-4519-b480-3a5c33b8c21b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Before training, the dataset undergoes extensive preprocessing: handling missing values, correcting sensor errors, encoding categorical variables, and scaling numerical features. Additional derived features (e.g., acceleration magnitude, Euler angles, angular velocity) are engineered to enhance the signal representation.","metadata":{}},{"cell_type":"code","source":"class CMISequenceDataset(Dataset):\n    \"\"\"\n    Dataset for CMI sensor data sequences. Expects a CSV with columns including:\n    'sequence_id', sensor features, and label.\n    \"\"\"\n    def __init__(self, df, is_train=True):\n        \"\"\"\n        df: DataFrame containing all rows (time-steps) with columns including 'sequence_id'.\n        is_train: whether labels are present.\n        \"\"\"\n        self.is_train = is_train\n\n        # Determine feature cols\n        self.feature_cols = [c for c in df.columns if c not in ['sequence_id', 'gesture']]\n        \n        # Determine label column\n        label_col = 'gesture'\n        \n        # Group rows by sequence_id\n        self.seqs = []\n        self.labels = []\n        for seq_id, group in df.groupby('sequence_id'):            \n            # Extract feature array for this sequence\n            seq_features = group[self.feature_cols].values.astype(np.float32)\n            self.seqs.append(seq_features)\n            \n            # If training, get the label for this sequence (assumed constant per sequence)\n            if self.is_train:\n                label = group[label_col].iloc[0]\n                self.labels.append(label)\n        \n    def __len__(self):\n        return len(self.seqs)\n    \n    def __getitem__(self, idx):\n        seq = self.seqs[idx]  # shape (time_steps, num_features)\n        \n        if np.isnan(seq).any():\n            print(f\"NaN found in sequence index {idx}\")\n\n        if self.is_train:\n            label = self.labels[idx]\n            return torch.from_numpy(seq), torch.tensor(label, dtype=torch.long)\n        else:\n            return torch.from_numpy(seq), None\n\n\ndef collate_fn(batch):\n    \"\"\"\n    Collate function to pad sequences to the same length within a batch.\n    Returns:\n        padded_seqs: tensor of shape (batch_size, max_len, num_features)\n        labels: tensor of shape (batch_size) or None for test\n        mask: boolean mask (batch_size, max_len) with True at padded positions\n    \"\"\"\n    sequences, labels = zip(*batch)\n    lengths = [seq.shape[0] for seq in sequences]\n    max_len = max(lengths)\n    # Number of features (all sequences have same feature count)\n    num_features = sequences[0].shape[1]\n    batch_size = len(sequences)\n    padded = torch.zeros(batch_size, max_len, num_features, dtype=torch.float32)\n    mask = torch.ones(batch_size, max_len, dtype=torch.bool)  # True indicates padding\n    for i, seq in enumerate(sequences):\n        length = seq.shape[0]\n        padded[i, :length, :] = seq\n        mask[i, :length] = False  # False for actual data\n    if labels[0] is not None:\n        labels = torch.tensor(labels, dtype=torch.long)\n    else:\n        labels = None\n    return padded, labels, mask","metadata":{"_uuid":"30e41b90-fa63-4127-919d-650a75fa5815","_cell_guid":"283f8218-b5df-452d-8f13-ea9ffa7d0aee","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:26.317507Z","iopub.execute_input":"2025-08-20T14:09:26.317713Z","iopub.status.idle":"2025-08-20T14:09:26.327068Z","shell.execute_reply.started":"2025-08-20T14:09:26.317696Z","shell.execute_reply":"2025-08-20T14:09:26.326249Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pre_processing(df, is_train=True, encoders=None, scaler=None):\n    # Drop irrelevant columns\n    cols_to_drop = ['row_id', 'sequence_type', 'behavior', 'subject', 'orientation', 'phase']\n    df = df.drop(columns=[col for col in cols_to_drop if col in df.columns])\n\n    # Section to deal with the missing values ------------------------------------------------------\n    # Replace NaNs with -1 for all tof_* columns\n    tof_cols = [c for c in df.columns if c.startswith(\"tof_\")]\n    df[tof_cols] = df[tof_cols].fillna(-1)\n    \n    # thm columns also have nan, we will replace with a mean value\n    thm_cols = [c for c in df.columns if c.startswith(\"thm_\")]\n    df[thm_cols] = df[thm_cols].fillna(df[thm_cols].mean())\n    \n    # rot columns have nan, we will fill with the a null quaternion {1, 0, 0, 0}\n    rot_cols = [c for c in df.columns if c.startswith(\"rot_\")]\n    rot_fill = {'rot_w': 1.0, 'rot_x': 0.0, 'rot_y': 0.0, 'rot_z': 0.0}\n    df[rot_cols] = df[rot_cols].fillna(rot_fill)\n    \n    # Section to deal with the erroneous sensor values ---------------------------------------------\n    # Identify sequences with any thm values < 10\n    threshold = 10\n    mask = (df.filter(like='thm_') < threshold).any(axis=1)\n    filtered_df = df[mask]\n    seqs_with_low_values = list(filtered_df['sequence_id'].unique())\n\n    # Fix thm_3 if all values are 0 in the sequence\n    for seq_id in seqs_with_low_values:\n        seq_mask = df['sequence_id'] == seq_id\n        if (df.loc[seq_mask, 'thm_3'] < threshold).all():\n            df.loc[seq_mask, 'thm_3'] = df.loc[seq_mask, 'thm_2']\n        if (df.loc[seq_mask, 'thm_1'] < threshold).all():\n            df.loc[seq_mask, 'thm_1'] = df.loc[seq_mask, 'thm_2']\n        \n    # Find categorical columns to encode\n    categorical_cols = df.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n    \n    if is_train:\n        encoders = {}\n        for col in categorical_cols:\n            if col == 'sequence_id':\n                le = LabelEncoder()\n                df[col] = le.fit_transform(df[col])\n            else:\n                le = LabelEncoder()\n                df[col] = le.fit_transform(df[col])        \n                encoders[col] = le\n    elif encoders is not None and not is_train:\n        for col in categorical_cols:\n            if col == 'sequence_id':\n                le = LabelEncoder()\n                df[col] = le.fit_transform(df[col])\n            else:\n                le = encoders[col]\n                df[col] = le.transform(df[col])\n    \n    # Identify sensor columns (customise this pattern if needed)\n    sensor_prefixes = ['acc_', 'thm_', 'tof_', 'rot_']\n    cols_to_scale = [c for c in df.columns if any(c.startswith(p) for p in sensor_prefixes)]\n    cols_to_scale.extend(['age', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm'])\n\n    # Compute all new features\n    acc_mag = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n    rot_mag = np.sqrt(df['rot_x']**2 + df['rot_y']**2 + df['rot_z']**2)\n    \n    w, x, y, z = df['rot_w'], df['rot_x'], df['rot_y'], df['rot_z']\n    euler_roll = np.arctan2(2 * (w * x + y * z), 1 - 2 * (x**2 + y**2))\n    euler_pitch = np.arcsin(np.clip(2 * (w * y - z * x), -1.0, 1.0))\n    euler_yaw = np.arctan2(2 * (w * z + x * y), 1 - 2 * (y**2 + z**2))\n    \n    # Grouped angular velocity (recommended)\n    angular_velocity_x = df.groupby('sequence_id')['rot_x'].diff().fillna(0)\n    angular_velocity_y = df.groupby('sequence_id')['rot_y'].diff().fillna(0)\n    angular_velocity_z = df.groupby('sequence_id')['rot_z'].diff().fillna(0)\n    \n    # Combine all new features into one DataFrame\n    new_features = pd.DataFrame({\n        'acc_mag': acc_mag,\n        'rot_mag': rot_mag,\n        'euler_roll': euler_roll,\n        'euler_pitch': euler_pitch,\n        'euler_yaw': euler_yaw,\n        'angular_velocity_x': angular_velocity_x,\n        'angular_velocity_y': angular_velocity_y,\n        'angular_velocity_z': angular_velocity_z\n    })\n\n    cols_to_scale.extend(['acc_mag', 'rot_mag', 'euler_roll', 'euler_pitch', 'euler_yaw', 'angular_velocity_x', 'angular_velocity_y', 'angular_velocity_z'])\n\n    # Concatenate all at once\n    df = pd.concat([df, new_features], axis=1).copy()\n\n    # Fit scaler on training split only\n    if is_train:\n        scaler = StandardScaler()\n        scaler.fit(df[cols_to_scale])\n    \n    # Apply to all splits\n    df[cols_to_scale] = scaler.transform(df[cols_to_scale])\n            \n    return df, encoders, scaler","metadata":{"_uuid":"074c514e-c6c5-4270-a2e6-aa6dbbe2bccb","_cell_guid":"8cdc70f3-8c14-4f26-8c23-68025fe47506","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:26.329079Z","iopub.execute_input":"2025-08-20T14:09:26.329280Z","iopub.status.idle":"2025-08-20T14:09:26.345191Z","shell.execute_reply.started":"2025-08-20T14:09:26.329264Z","shell.execute_reply":"2025-08-20T14:09:26.344403Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, encoders, scaler = pre_processing(train_df)\ntest_df, _, _ = pre_processing(test_df, is_train=False, encoders=encoders, scaler=scaler)","metadata":{"_uuid":"f6031949-d80f-46cc-bdd1-23efacb584b9","_cell_guid":"62120960-f695-4915-b58e-f550bb99edfd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:26.346001Z","iopub.execute_input":"2025-08-20T14:09:26.346248Z","iopub.status.idle":"2025-08-20T14:09:43.104430Z","shell.execute_reply.started":"2025-08-20T14:09:26.346227Z","shell.execute_reply":"2025-08-20T14:09:43.103818Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split train into train/validation sets (e.g., 80/20 split)\ntrain_df_part, val_df = train_test_split(train_df, test_size=0.2, random_state=42)","metadata":{"_uuid":"ee6de295-7868-403b-bcc9-9065ff4296ce","_cell_guid":"ae2248e8-08f3-4d74-a8b2-b1ff25cfaaa5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:43.105135Z","iopub.execute_input":"2025-08-20T14:09:43.105399Z","iopub.status.idle":"2025-08-20T14:09:46.111937Z","shell.execute_reply.started":"2025-08-20T14:09:43.105377Z","shell.execute_reply":"2025-08-20T14:09:46.111381Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create datasets and data loaders\ntrain_dataset = CMISequenceDataset(train_df_part, is_train=True)\n# Use the same label encoder for validation and test to ensure consistent class mapping\nval_dataset = CMISequenceDataset(val_df, is_train=True)\n# For test set, we do not have labels\ntest_dataset = CMISequenceDataset(test_df, is_train=False)\n\nbatch_size = 32\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, \n                          collate_fn=collate_fn, drop_last=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, \n                        collate_fn=collate_fn)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, \n                         collate_fn=collate_fn)","metadata":{"_uuid":"18d1f51b-4e35-4d08-a399-adaf84cfd206","_cell_guid":"1ec2b030-2395-4725-9922-2311fca82eab","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:09:46.112685Z","iopub.execute_input":"2025-08-20T14:09:46.112902Z","iopub.status.idle":"2025-08-20T14:11:57.813316Z","shell.execute_reply.started":"2025-08-20T14:09:46.112885Z","shell.execute_reply":"2025-08-20T14:11:57.812774Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1D-CNN - Transformer: Model Defnition and training","metadata":{"_uuid":"4d6d1eb2-cd35-4140-9380-706bbfbfe419","_cell_guid":"789156ee-d8a9-4225-9de8-bbf9840e17b9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"We define a hybrid model combining 1D convolutional layers (to capture local temporal patterns) with a Transformer encoder (to model long-range dependencies). The model is trained and validated using PyTorch, with metrics such as accuracy and F1-score to track performance.","metadata":{}},{"cell_type":"code","source":"class CNNTransformer(nn.Module):\n    \"\"\"\n    A lightweight CNN + Transformer model for time-series classification.\n    \"\"\"\n    def __init__(self, input_dim, num_classes, hidden_dim=128, n_heads=4, n_layers=2, dropout=0.3):\n        super(CNNTransformer, self).__init__()\n        # 1D CNN layers to preprocess the time-series (fold time steps).\n        # Input shape to CNN: (batch, input_dim, seq_len)\n        self.conv1 = nn.Conv1d(in_channels=input_dim, out_channels=hidden_dim, \n                               kernel_size=3, stride=2, padding=1)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(p=dropout)\n        # Transformer Encoder layers\n        encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_dim, nhead=n_heads, \n                                                   dim_feedforward=hidden_dim*4, \n                                                   activation='relu')\n        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=n_layers)\n        # Classification head\n        self.classifier = nn.Linear(hidden_dim, num_classes)\n        \n    def forward(self, x, src_key_padding_mask=None):\n        \"\"\"\n        x: tensor of shape (batch_size, seq_len, input_dim)\n        src_key_padding_mask: bool mask (batch_size, seq_len) True for padding\n        \"\"\"\n        # Prepare for conv: (batch, input_dim, seq_len)\n        x = x.permute(0, 2, 1)\n        x = self.relu(self.conv1(x))  # downsample time by ~2\n        x = self.dropout(x)\n        # x now (batch, hidden_dim, new_seq_len)\n        # Prepare for transformer: (new_seq_len, batch, hidden_dim)\n        x = x.permute(2, 0, 1)\n        # Adjust mask for new length\n        if src_key_padding_mask is not None:\n            # Original mask (batch, seq_len) -> after conv pooling, we create a new mask.\n            # For simplicity, create no further mask (this is an approximation).\n            # One could recompute mask by checking if the entire pooled input was padding.\n            new_mask = None\n        else:\n            new_mask = None\n        # Transformer encoding\n        x = self.transformer(x, src_key_padding_mask=new_mask)\n        # x shape: (new_seq_len, batch, hidden_dim) -> (batch, new_seq_len, hidden_dim)\n        x = x.permute(1, 0, 2)\n        # Mean pooling over time dimension (ignoring any padding by ~mask)\n        # For simplicity, we'll take the average of all time steps (more elaborate masking possible).\n        x = x.mean(dim=1)  # shape (batch, hidden_dim)\n        logits = self.classifier(x)  # shape (batch, num_classes)\n        return logits","metadata":{"_uuid":"e4730c94-5d6d-4ada-aa54-1ced055e7e9e","_cell_guid":"ceec24e6-df22-4a35-81f9-4c27b274734d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:11:57.814009Z","iopub.execute_input":"2025-08-20T14:11:57.814207Z","iopub.status.idle":"2025-08-20T14:11:57.821262Z","shell.execute_reply.started":"2025-08-20T14:11:57.814191Z","shell.execute_reply":"2025-08-20T14:11:57.820433Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, criterion, optimizer, device):\n    model.train()\n    total_loss = 0.0\n    all_preds = []\n    all_labels = []\n    for batch in loader:\n        seqs, labels, mask = batch\n        seqs = seqs.to(device)\n        labels = labels.to(device)\n        # Forward pass\n        logits = model(seqs)\n        loss = criterion(logits, labels)\n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        optimizer.step()\n        total_loss += loss.item() * seqs.size(0)\n        # Track training metrics\n        preds = logits.argmax(dim=1).cpu().numpy()\n        all_preds.extend(preds.tolist())\n        all_labels.extend(labels.cpu().numpy().tolist())\n    avg_loss = total_loss / len(loader.dataset)\n    acc = accuracy_score(all_labels, all_preds)\n    return avg_loss, acc\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0.0\n    all_preds = []\n    all_labels = []\n    with torch.no_grad():\n        for batch in loader:\n            seqs, labels, mask = batch\n            seqs = seqs.to(device)\n            labels = labels.to(device)\n            logits = model(seqs)\n            loss = criterion(logits, labels)\n            total_loss += loss.item() * seqs.size(0)\n            preds = logits.argmax(dim=1).cpu().numpy()\n            all_preds.extend(preds.tolist())\n            all_labels.extend(labels.cpu().numpy().tolist())\n    avg_loss = total_loss / len(loader.dataset)\n    acc = accuracy_score(all_labels, all_preds)\n    # Also compute F1 as an example (macro)\n    f1 = f1_score(all_labels, all_preds, average='macro')\n    return avg_loss, acc, f1","metadata":{"_uuid":"94ec834a-89d8-4fad-a9c5-10e494ce25a8","_cell_guid":"4824664e-4103-4f2b-9143-051538220a26","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:11:57.822103Z","iopub.execute_input":"2025-08-20T14:11:57.822410Z","iopub.status.idle":"2025-08-20T14:11:57.851212Z","shell.execute_reply.started":"2025-08-20T14:11:57.822382Z","shell.execute_reply":"2025-08-20T14:11:57.850539Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model setup\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ninput_dim = len(train_dataset.feature_cols)\nnum_classes = len(set(train_dataset.labels))\nmodel = CNNTransformer(input_dim=input_dim, num_classes=num_classes).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Optionally, add a learning rate scheduler\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n\nnum_epochs = 30\nbest_val_f1 = 0.0\nfor epoch in range(num_epochs):\n    train_loss, train_acc = train_one_epoch(model, train_loader, criterion, optimizer, device)\n    val_loss, val_acc, val_f1 = validate(model, val_loader, criterion, device)\n    print(f\"Epoch {epoch+1}/{num_epochs}: \"\n          f\"Train Loss={train_loss:.4f}, Acc={train_acc:.4f} | \"\n          f\"Val Loss={val_loss:.4f}, Acc={val_acc:.4f}, F1={val_f1:.4f}\")\n    if val_f1 > best_val_f1:\n        best_val_f1 = val_f1\n    scheduler.step()\n\n# Inference on test set\nmodel.eval()\nsequence_ids = []\npreds_labels = []\nwith torch.no_grad():\n    for batch_idx, batch in enumerate(test_loader):\n        seqs, _, mask = batch\n        seqs = seqs.to(device)\n        logits = model(seqs, src_key_padding_mask=mask.to(device))\n        preds = logits.argmax(dim=1).cpu().numpy()\n        preds_labels.extend(preds.tolist())\n        # Collect sequence IDs for this batch\n        # We need to map batch order to sequence IDs; here we assume test_dataset.seqs preserves order of test_df\n        batch_ids = test_df['sequence_id'].unique()[batch_idx*test_loader.batch_size : (batch_idx+1)*test_loader.batch_size]\n        sequence_ids.extend(batch_ids.tolist())","metadata":{"_uuid":"3e3978e4-3645-4fea-9dbf-893f3542e1f0","_cell_guid":"240def89-e02c-467e-9345-6ac4067b58ae","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:11:57.851994Z","iopub.execute_input":"2025-08-20T14:11:57.852244Z","iopub.status.idle":"2025-08-20T14:14:12.653694Z","shell.execute_reply.started":"2025-08-20T14:11:57.852221Z","shell.execute_reply":"2025-08-20T14:14:12.653027Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"_uuid":"92d39ecb-3a8c-459d-b1d6-36e3dbaf3ac5","_cell_guid":"78efefcb-ee1e-40c4-b289-5662abfbf8db","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Finally, predictions are generated for the test set and mapped back to the original gesture labels. A submission DataFrame is prepared in the required Kaggle format, ensuring compatibility with the competition’s evaluation pipeline.","metadata":{}},{"cell_type":"code","source":"# Map numeric predictions back to gesture names\ninv_label_map = {i: lbl for i, lbl in enumerate(encoders['gesture'].classes_)}\npredicted_gestures = [inv_label_map.get(label, \"Non-Target\") for label in preds_labels]\nsubmission = pd.DataFrame({\n    \"sequence_id\": sequence_ids,\n    \"gesture\": predicted_gestures\n})\n\nsubmission","metadata":{"_uuid":"946967f8-7c28-4761-96b9-035d531dbb8c","_cell_guid":"acdb59af-e7d6-4b0d-95d5-1ca7ee7cc369","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:14:12.654493Z","iopub.execute_input":"2025-08-20T14:14:12.654917Z","iopub.status.idle":"2025-08-20T14:14:12.663094Z","shell.execute_reply.started":"2025-08-20T14:14:12.654900Z","shell.execute_reply":"2025-08-20T14:14:12.662384Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    # Convert polars to pandas\n    seq_df = sequence.to_pandas()\n    demographics_df = demographics.to_pandas()\n\n    seq_df = seq_df.merge(demographics_df, on=\"subject\", how=\"left\")\n\n    X, _, _ = pre_processing(seq_df, is_train=False, encoders=encoders, scaler=scaler)\n    \n    # drop unnecessary columns\n    cols_to_drop = ['sequence_id']\n    X = X.drop(columns=cols_to_drop)\n\n    X = X.values.astype(np.float32)\n    \n    # Convert to torch tensor\n    x_tensor = torch.tensor(X).unsqueeze(0).to(device)\n    mask = torch.zeros(1, x_tensor.shape[1], dtype=torch.bool).to(device)\n    \n    # Inference\n    model.eval()\n    with torch.no_grad():\n        logits = model(x_tensor, src_key_padding_mask=mask)\n        pred_idx = torch.argmax(logits, dim=1).item()\n    \n    # Map back to label string\n    pred_class = encoders['gesture'].inverse_transform([pred_idx])[0]\n    return pred_class","metadata":{"_uuid":"19a7f136-ad30-4658-8f6c-918626acac69","_cell_guid":"5b00a0ba-639c-4d50-9d0e-7c2ebaf77a0b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:14:12.663928Z","iopub.execute_input":"2025-08-20T14:14:12.664187Z","iopub.status.idle":"2025-08-20T14:14:12.675528Z","shell.execute_reply.started":"2025-08-20T14:14:12.664167Z","shell.execute_reply":"2025-08-20T14:14:12.674846Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport kaggle_evaluation.cmi_inference_server\n\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"_uuid":"925e511e-92f8-4f31-b2b4-7ec4f473d63d","_cell_guid":"5235d5b8-9f36-4473-b1d7-84559b25046d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-08-20T14:20:48.626854Z","iopub.execute_input":"2025-08-20T14:20:48.627747Z","iopub.status.idle":"2025-08-20T14:20:49.778871Z","shell.execute_reply.started":"2025-08-20T14:20:48.627716Z","shell.execute_reply":"2025-08-20T14:20:49.778265Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}