{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-20T12:27:06.328822Z","iopub.execute_input":"2025-12-20T12:27:06.329905Z","iopub.status.idle":"2025-12-20T12:27:06.343572Z","shell.execute_reply.started":"2025-12-20T12:27:06.329860Z","shell.execute_reply":"2025-12-20T12:27:06.342124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def engineer_therm_features(tmp_array):\n    \"\"\"\n    tmp_array: numpy array or torch tensor of shape (..., 5)\n        expected order: [TMP1, TMP2, TMP3, TMP4, TMP5]\n    returns:\n        engineered features array (..., 8) = 5 raw + 3 engineered\n    \"\"\"\n\n    TMP1 = tmp_array[..., 0]\n    TMP2 = tmp_array[..., 1]\n    TMP3 = tmp_array[..., 2]\n    TMP4 = tmp_array[..., 3]\n    TMP5 = tmp_array[..., 4]\n\n    # engineered features\n    hand_proximity = TMP2 - TMP1\n    left_right = TMP3 - TMP5\n    arm_delta = TMP1 - TMP4\n\n    engineered = torch.stack([\n        TMP1, TMP2, TMP3, TMP4, TMP5,\n        hand_proximity,\n        left_right,\n        arm_delta,\n    ], dim=-1)\n\n    return engineered\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass ThermEncoder(nn.Module):\n    def __init__(self, k_classes=8):\n        super().__init__()\n        \n        self.mlp = nn.Sequential(\n            nn.Linear(8, 16),\n            nn.ReLU(),\n            nn.Linear(16, 8),\n            nn.ReLU(),\n            nn.Linear(8, k_classes)  # logits in latent k-space\n        )\n        \n    def forward(self, x):\n        \"\"\"\n        x: (batch, 5) raw therm readings\n        returns logits: (batch, k_classes)\n        \"\"\"\n        # convert raw → engineered\n        x = engineer_therm_features(x)\n        \n        logits = self.mlp(x)\n        return logits\n\nimport torch\nfrom torch.utils.data import Dataset\nimport numpy as np\n\nclass ThermWindowDataset(Dataset):\n    def __init__(\n        self,\n        df,\n        window_size=20,\n        stride=5,\n        feature_cols=(\"thm_1\", \"thm_1\", \"thm_1\", \"thm_1\", \"thm_1\"),\n        label_col=\"gesture_id\"\n    ):\n        self.samples = []\n\n        for seq_id, seq_df in df.groupby(\"sequence_id\"):\n            data = seq_df[feature_cols].values   # (L, 5)\n            labels = seq_df[label_col].values\n\n            L = len(seq_df)\n            for start in range(0, L - window_size + 1, stride):\n                end = start + window_size\n\n                x = data[start:end].T                  # (5, T)\n                y = labels[start:end]\n\n                target = np.bincount(y).argmax()\n\n                self.samples.append((\n                    torch.tensor(x, dtype=torch.float32),\n                    torch.tensor(target, dtype=torch.long)\n                ))\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, idx):\n        return self.samples[idx]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-20T12:27:06.345312Z","iopub.execute_input":"2025-12-20T12:27:06.345586Z","iopub.status.idle":"2025-12-20T12:27:06.368332Z","shell.execute_reply.started":"2025-12-20T12:27:06.345562Z","shell.execute_reply":"2025-12-20T12:27:06.367278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FLEXIBLE HYBRID CNN + TRANSFORMER MODEL FOR KAGGLE\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom scipy.spatial.transform import Rotation as R\nimport time\nfrom sklearn.preprocessing import StandardScaler\n\n# ============================================================\n# CONFIGURABLE VARIABLES\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nWINDOW_BEFORE = 10\nWINDOW_AFTER = 10\nBATCH_SIZE = 32\nEPOCHS = 20\nLR = 3e-4\nD_MODEL = 64\nAXIS_HIDDEN_DIM = 32\nUSE_AXISCNN = True\nUSE_TRANSFORMER = True\nFUSION = 'concat'  # 'concat' or 'gated'\nDROPOUT = 0.1\nUSE_ACC = True\nUSE_ROT = True\nVAL_RATIO = 0.2\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nSEED = 42\n\n# ============================================================\n# HELPER FUNCTIONS\n# ============================================================\n\ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    rot_cleaned = rot_data.copy()\n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        if missing_count == 0:\n            norm = np.linalg.norm(row)\n            rot_cleaned[i] = row / norm if norm > 1e-8 else [1.0,0.0,0.0,0.0]\n        elif missing_count == 1:\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            sum_squares = np.sum(valid_values**2)\n            if sum_squares <= 1.0:\n                missing_value = np.sqrt(max(0,1.0-sum_squares))\n                if i>0 and not np.isnan(rot_cleaned[i-1,missing_idx]):\n                    if rot_cleaned[i-1,missing_idx]<0: missing_value=-missing_value\n                rot_cleaned[i,missing_idx]=missing_value\n                rot_cleaned[i,~np.isnan(row)] = valid_values\n            else: rot_cleaned[i]=[1.0,0.0,0.0,0.0]\n        else:\n            rot_cleaned[i]=[1.0,0.0,0.0,0.0]\n    return rot_cleaned\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    try:\n        rot_scipy = rot[:,[1,2,3,0]]\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        mask = norms<1e-8\n        rot_scipy[mask] = [0.0,0.0,0.0,1.0]\n        r = R.from_quat(rot_scipy)\n        return r.apply(acc)\n    except:\n        return acc.copy()\n\ndef remove_gravity_from_acc(acc_data, rot_data):\n    num_samples = acc_data.shape[0]\n    linear_accel = np.zeros_like(acc_data)\n    gravity_world = np.array([0,0,9.81])\n    for i in range(num_samples):\n        if np.all(np.isnan(rot_data[i])) or np.all(np.isclose(rot_data[i],0)):\n            linear_accel[i,:] = acc_data[i,:]\n            continue\n        try:\n            rotation = R.from_quat(rot_data[i])\n            gravity_sensor_frame = rotation.apply(gravity_world,inverse=True)\n            linear_accel[i,:] = acc_data[i,:]-gravity_sensor_frame\n        except:\n            linear_accel[i,:] = acc_data[i,:]\n    return linear_accel\n\ndef calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n    num_samples = rot_data.shape[0]\n    angular_vel = np.zeros((num_samples,3))\n    for i in range(num_samples-1):\n        q_t = rot_data[i]; q_t_plus = rot_data[i+1]\n        if np.any(np.isnan(q_t)) or np.any(np.isnan(q_t_plus)): continue\n        try:\n            r1 = R.from_quat(q_t); r2 = R.from_quat(q_t_plus)\n            delta_rot = r1.inv()*r2\n            angular_vel[i,:] = delta_rot.as_rotvec()/time_delta\n        except: continue\n    return angular_vel\n\ndef calculate_angular_distance(rot_data):\n    num_samples = rot_data.shape[0]\n    angular_dist = np.zeros(num_samples)\n    for i in range(num_samples-1):\n        q1=rot_data[i]; q2=rot_data[i+1]\n        if np.any(np.isnan(q1)) or np.any(np.isnan(q2)): continue\n        try:\n            r1=R.from_quat(q1); r2=R.from_quat(q2)\n            rel_rot = r1.inv()*r2\n            angular_dist[i]=np.linalg.norm(rel_rot.as_rotvec())\n        except: continue\n    return angular_dist\n\n# ============================================================\n# DATASET\n# ============================================================\nclass AccRotHybridDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n\n    def __init__(self, df, window_before=10, window_after=10, normalize_tof=True):\n        #self.df = pd.read_csv(csv_path)\n        self.df = df\n        self.window_before = window_before\n        self.window_after = window_after\n        self.window_len = window_before + window_after + 1\n        self.normalize_tof = normalize_tof\n\n        # -----------------------\n        # Gesture encoding\n        # -----------------------\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # -----------------------\n        # ToF columns\n        # -----------------------\n        self.tof_cols = [c for c in self.df.columns if c.startswith(\"tof_\")]\n        self.tof_dim = len(self.tof_cols)  # This is important!\n\n        # -----------------------\n        # Optional ToF normalization\n        tof_data = self.df[self.tof_cols]\n        X = tof_data.values.astype(np.float32)\n        # -----------------------\n        if self.normalize_tof:\n            self.tof_scaler = StandardScaler()\n            X = self.tof_scaler.fit_transform(X)\n            X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\n        \n        self.df[self.tof_cols] = X\n\n        # -----------------------\n        # Therm\n        # -----------------------\n        self.therm_cols = [c for c in self.df.columns if c.startswith(\"thm_\")]\n        \n        # -----------------------\n        # Build windows\n        # -----------------------\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        self.windows = []\n\n        for _, seq in self.sequences:\n            seq = seq.reset_index(drop=True)\n\n            gesture_rows = seq[seq[\"phase\"] == \"Gesture\"]\n            if gesture_rows.empty:\n                continue\n\n            center = gesture_rows.index[0]\n            start = center - self.window_before\n            end = center + self.window_after + 1\n\n            start_c = max(0, start)\n            end_c = min(len(seq), end)\n            \n            # -----------------------\n            # ACC / ROT\n            # -----------------------\n            acc = seq.iloc[start_c:end_c][self.ACC_COLS].values.astype(np.float32)\n            rot = seq.iloc[start_c:end_c][self.ROT_COLS].values.astype(np.float32)\n            tof = seq.iloc[start_c:end_c][self.tof_cols].values.astype(np.float32)\n\n            rot = handle_quaternion_missing_values(rot)\n            norms = np.linalg.norm(rot, axis=1, keepdims=True)\n            norms[norms < 1e-6] = 1.0\n            rot = rot / norms\n\n            acc_world = compute_world_acceleration(acc, rot)\n            acc_linear = remove_gravity_from_acc(acc, rot)\n            angular_vel = calculate_angular_velocity_from_quat(rot)\n            angular_dist = calculate_angular_distance(rot)\n\n            acc_features = np.concatenate(\n                [acc_world, acc_linear, angular_vel, angular_dist[:, None]], axis=1\n            )\n\n            therm = seq.iloc[start_c:end_c][self.therm_cols].values.astype(np.float32)\n            \n            # -----------------------\n            # Padding\n            # -----------------------\n            acc_pad = np.zeros((self.window_len, acc_features.shape[1]), dtype=np.float32)\n            rot_pad = np.zeros((self.window_len, 4), dtype=np.float32)\n            tof_pad = np.zeros((self.window_len, self.tof_dim), dtype=np.float32)\n            mask = np.zeros(self.window_len, dtype=np.bool_)\n\n            insert = start_c - start\n            actual_len = end_c - start_c\n            \n            acc_pad[insert:insert + len(acc_features)] = acc_features\n            rot_pad[insert:insert + len(rot)] = rot\n            mask[insert:insert + len(acc_features)] = True\n            tof_pad[insert:insert + actual_len] = tof  # Pad ToF too\n\n            therm_pad = np.zeros((self.window_len, 5), dtype=np.float32)\n            therm_pad[insert:insert + actual_len] = therm\n\n            # -----------------------\n            # Label\n            # -----------------------\n            label = seq[\"gesture_id\"].mode().iloc[0]\n\n            #self.windows.append((acc_pad, rot_pad, tof_pad, mask, label))\n            self.windows.append((acc_pad, rot_pad, tof_pad, therm_pad, mask, label))\n\n        print(f\"Created {len(self.windows)} multimodal windows\")\n        print(f\"ToF dimension: {self.tof_dim}\")  # Debug print\n\n    def __len__(self):\n        return len(self.windows)\n\n    def __getitem__(self, idx):\n        acc, rot, tof, therm, mask, label = self.windows[idx]\n\n        if not USE_ACC:\n            acc = np.zeros_like(acc, dtype=np.float32)\n        if not USE_ROT:\n            rot = np.zeros_like(rot, dtype=np.float32)\n\n        return {\n            \"acc\": torch.tensor(acc, dtype=torch.float32),\n            \"rot\": torch.tensor(rot, dtype=torch.float32),\n            \"tof\": torch.tensor(tof, dtype=torch.float32),\n            \"mask\": torch.tensor(mask, dtype=torch.bool),\n            \"label\": torch.tensor(label, dtype=torch.long),\n            \"therm\": torch.tensor(therm, dtype=torch.float32)\n        }\n\n# ============================================================\n# MODEL\n# ============================================================\n\nclass AxisCNN(nn.Module):\n    def __init__(self, hidden_dim=32):\n        super().__init__()\n        self.hidden_dim = hidden_dim\n        self.net = nn.Sequential(\n            nn.Conv1d(1, hidden_dim, kernel_size=5, padding=2),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Conv1d(hidden_dim, hidden_dim, kernel_size=5, padding=2),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Conv1d(hidden_dim, hidden_dim, kernel_size=3, padding=1),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n        )\n    def forward(self,x):\n        x = self.net(x)\n        x = x.mean(dim=-1)\n        return x\n\nclass EnhancedCNNEncoder(nn.Module):\n    def __init__(self, input_dim, d_model=64, axis_hidden_dim=32):\n        super().__init__()\n        self.axis_cnns = nn.ModuleList([AxisCNN(hidden_dim=axis_hidden_dim) for _ in range(input_dim)])\n        self.axis_proj = nn.Linear(input_dim*axis_hidden_dim, d_model)\n        self.temporal_cnn = nn.Sequential(\n            nn.Conv1d(d_model,d_model,3,padding=1), nn.BatchNorm1d(d_model), nn.ReLU(),\n            nn.Conv1d(d_model,d_model,3,padding=1), nn.BatchNorm1d(d_model), nn.ReLU()\n        )\n        self.ln = nn.LayerNorm(d_model)\n    def forward(self,x):\n        B,T,F = x.shape\n        feats = []\n        for i in range(F):\n            axis_data = x[:,:,i:i+1].transpose(1,2)\n            feats.append(self.axis_cnns[i](axis_data))\n        projected = self.axis_proj(torch.cat(feats,dim=1))\n        expanded = projected.unsqueeze(1).repeat(1,T,1)\n        expanded = self.temporal_cnn(expanded.transpose(1,2)).transpose(1,2)\n        return self.ln(expanded)\n        \n# class ToFAutoencoder(nn.Module):\n#     def __init__(self, input_dim, latent_dim=8, hidden_dim=128):\n#         super().__init__()\n\n#         self.encoder = nn.Sequential(\n#             nn.Linear(input_dim, hidden_dim),\n#             nn.ReLU(),\n#             nn.Linear(hidden_dim, hidden_dim // 2),\n#             nn.ReLU(),\n#             nn.Linear(hidden_dim // 2, latent_dim),\n#         )\n\n#         self.decoder = nn.Sequential(\n#             nn.Linear(latent_dim, hidden_dim // 2),\n#             nn.ReLU(),\n#             nn.Linear(hidden_dim // 2, hidden_dim),\n#             nn.ReLU(),\n#             nn.Linear(hidden_dim, input_dim),\n#         )\n\n#     def forward(self, x):\n#         z = self.encoder(x)\n#         recon = self.decoder(z)\n#         return recon, z\nclass TemporalToFAutoencoder(nn.Module):\n    def __init__(self, input_dim, seq_len, latent_dim=8, hidden_dim=128):\n        super().__init__()\n        \n        # Encoder: [B, T, F] -> [B, latent_dim]\n        self.encoder = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(seq_len * input_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, latent_dim),\n        )\n        \n        # Decoder: [B, latent_dim] -> [B, T, F]\n        self.decoder = nn.Sequential(\n            nn.Linear(latent_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, seq_len * input_dim),\n        )\n        self.seq_len = seq_len\n        self.input_dim = input_dim\n        \n    def forward(self, x):\n        B, T, F = x.shape\n        z = self.encoder(x)  # [B, latent_dim]\n        recon_flat = self.decoder(z)  # [B, T*F]\n        recon = recon_flat.view(B, self.seq_len, self.input_dim)  # [B, T, F]\n        return recon, z        \nclass TransformerEncoder(nn.Module):\n    def __init__(self,d_model=64,nhead=4,layers=2):\n        super().__init__()\n        enc_layer = nn.TransformerEncoderLayer(d_model=d_model,nhead=nhead,batch_first=True,dim_feedforward=d_model*4,activation='gelu')\n        self.encoder = nn.TransformerEncoder(enc_layer,layers)\n    def forward(self,x,mask):\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        m = mask.unsqueeze(-1).float()\n        return (x*m).sum(dim=1)/m.sum(dim=1).clamp(min=1)\n\nclass HybridFusedModel(nn.Module):\n    def __init__(self, num_classes, acc_feat_dim, rot_dim=4,\n                 tof_dim=None, tof_latent_dim=8, d_model=64):\n        super().__init__()\n        self.use_acc = USE_ACC\n        self.use_rot = USE_ROT\n        self.use_tof = USE_TOF and (tof_dim is not None)  # Fix: check USE_TOF flag\n        \n        self.acc_encoder = EnhancedCNNEncoder(acc_feat_dim, d_model) if USE_ACC else None\n        self.rot_encoder = EnhancedCNNEncoder(rot_dim, d_model) if USE_ROT else None\n        self.acc_trans = TransformerEncoder(d_model) if USE_TRANSFORMER and USE_ACC else None\n        self.rot_trans = TransformerEncoder(d_model) if USE_TRANSFORMER and USE_ROT else None\n        self.therm_encoder = ThermEncoder(k_classes=8)\n        self.therm_trans = TransformerEncoder(d_model)\n        k_classes=8\n        self.therm_proj = nn.Linear(k_classes, d_model)   # d_model = transformer dim\n        \n        # ToF autoencoder for temporal data\n        if self.use_tof:\n            self.tof_ae = TemporalToFAutoencoder(\n                input_dim=tof_dim, \n                seq_len=WINDOW_BEFORE + WINDOW_AFTER + 1,  # window_len\n                latent_dim=tof_latent_dim\n            )\n            self.tof_proj = nn.Linear(tof_latent_dim, d_model)\n            self.tof_scale = nn.Parameter(torch.tensor(0.1))\n        else:\n            self.tof_ae = None\n            self.tof_proj = None\n            self.tof_scale = None\n\n        # ------------------\n        # Fusion + Head\n        # ------------------\n        # Count only ACC and ROT outputs (ToF is fused into them)\n        #fusion_dim = d_model * (USE_ACC + USE_ROT)\n        fusion_dim = d_model * (USE_ACC + USE_ROT + 1)   # +1 for therm\n        \n        self.fusion = nn.Sequential(\n            nn.LayerNorm(fusion_dim),\n            nn.Linear(fusion_dim, d_model),\n            nn.GELU(),\n            nn.Dropout(DROPOUT)\n        )\n\n        self.head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model // 2),\n            nn.GELU(),\n            nn.Dropout(DROPOUT),\n            nn.Linear(d_model // 2, num_classes)\n        )\n\n    def forward(self, acc, rot, therm, mask, tof=None):\n        feats = []\n\n        # ======================\n        # ToF → token\n        # ======================\n        if self.use_tof and tof is not None:\n            _, tof_latent = self.tof_ae(tof)              # [B, L]\n            tof_token = self.tof_proj(tof_latent)         # [B, D]\n            tof_token = tof_token.unsqueeze(1)            # [B, 1, D]\n            tof_token = self.tof_scale * tof_token        # soft conditioning\n        else:\n            tof_token = None\n\n        # ======================\n        # ACC branch\n        # ======================\n        if self.use_acc:\n            acc_feat = self.acc_encoder(acc)              # [B, T, D]\n\n            if tof_token is not None:\n                acc_feat = torch.cat([tof_token, acc_feat], dim=1)\n                acc_mask = torch.cat(\n                    [torch.ones(mask.size(0), 1, device=mask.device, dtype=torch.bool), mask],\n                    dim=1\n                )\n            else:\n                acc_mask = mask\n\n            acc_out = self.acc_trans(acc_feat, acc_mask)\n            feats.append(acc_out)\n\n        # ======================\n        # ROT branch\n        # ======================\n        if self.use_rot:\n            rot_feat = self.rot_encoder(rot)\n\n            if tof_token is not None:\n                rot_feat = torch.cat([tof_token, rot_feat], dim=1)\n                rot_mask = torch.cat(\n                    [torch.ones(mask.size(0), 1, device=mask.device, dtype=torch.bool), mask],\n                    dim=1\n                )\n            else:\n                rot_mask = mask\n\n            rot_out = self.rot_trans(rot_feat, rot_mask)\n            feats.append(rot_out)\n\n        # ======================\n        # Therm\n        # ======================\n        #if self.use_therm and therm is not None:\n        # therm: [B, T, 5]\n        #therm_feat = self.therm_encoder(therm)     # [B, T, D] or [B, D]\n        #therm_out = self.therm_trans(therm_feat)   # if transformer-based\n        #feats.append(therm_out)\n\n        # therm : [B, 5, T]\n        #therm = therm.permute(0, 2, 1) # [B, T, 5]\n        B, T, _ = therm.shape\n        \n        therm_flat = therm.reshape(B*T, 5)\n        therm_logits = self.therm_encoder(therm_flat)\n        therm_feat = therm_logits.reshape(B, T, -1)   # [B, T, k]\n        therm_feat = self.therm_proj(therm_feat)      # [B, T, D]\n        therm_mask = mask\n        therm_out = self.therm_trans(therm_feat, therm_mask)\n        \n        feats.append(therm_out)\n\n        # ======================\n        # Fusion\n        # ======================\n        if len(feats) > 1:\n            z = self.fusion(torch.cat(feats, dim=1))\n        else:\n            z = feats[0]\n\n        return self.head(z)\n        \n# ============================================================\n# TRAINING\n# ============================================================\nWINDOW_BEFORE = 20\nWINDOW_AFTER = 20\nEPOCHS = 20\nUSE_AXISCNN = False\nUSE_TRANSFORMER = True\nFUSION = 'concat'  # 'concat' or 'gated'\nUSE_ACC = True\nUSE_ROT = True\nUSE_TOF = True\n\ntorch.manual_seed(SEED)\ndf = pd.read_csv(CSV_PATH)\ntherm_cols = [\"thm_1\",\"thm_2\",\"thm_3\",\"thm_4\",\"thm_5\"]\ndf[therm_cols] = df[therm_cols].fillna(0.0)\ndataset = AccRotHybridDataset(df,WINDOW_BEFORE,WINDOW_AFTER)\nnum_classes = len(dataset.gesture_classes)\nacc_feat_dim = dataset[0][\"acc\"].shape[1] if USE_ACC else 0\ntof_feat_dim = dataset.tof_dim if USE_TOF else 0\n\ntrain_size = int((1-VAL_RATIO)*len(dataset)); test_size=len(dataset)-train_size\ntrain_ds, test_ds = random_split(dataset,[train_size,test_size])\n\ntrain_loader = DataLoader(train_ds,batch_size=BATCH_SIZE,shuffle=True)\ntest_loader = DataLoader(test_ds,batch_size=BATCH_SIZE)\n\nmodel = HybridFusedModel(\n    num_classes=num_classes,\n    acc_feat_dim=acc_feat_dim,\n    tof_dim=tof_feat_dim ).to(DEVICE)\noptimizer = torch.optim.AdamW(model.parameters(),lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train(); total_loss=0\n    for batch in train_loader:\n        acc=batch[\"acc\"].to(DEVICE) if USE_ACC else None\n        rot=batch[\"rot\"].to(DEVICE) if USE_ROT else None\n        tof=batch[\"tof\"].to(DEVICE) if USE_TOF else None\n        therm = batch[\"therm\"].to(DEVICE)\n        mask=batch[\"mask\"].to(DEVICE); label=batch[\"label\"].to(DEVICE)\n        \n        optimizer.zero_grad()\n        logits = model(acc,rot,therm,mask,tof)\n        loss = criterion(logits,label)\n        loss.backward()\n        optimizer.step()\n        total_loss+=loss.item()\n    print(f\"Epoch {epoch+1} | Loss: {total_loss/len(train_loader):.4f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\n\nmodel.eval()\nall_preds, all_labels=[],[]\nwith torch.no_grad():\n    for batch in test_loader:\n        acc=batch[\"acc\"].to(DEVICE) if USE_ACC else None\n        rot=batch[\"rot\"].to(DEVICE) if USE_ROT else None\n        tof=batch[\"tof\"].to(DEVICE) if USE_TOF else None\n        mask=batch[\"mask\"].to(DEVICE)\n        \n        logits = model(acc,rot,therm,mask,tof)\n        preds = logits.argmax(dim=1)\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(batch[\"label\"].numpy())\n\nall_preds=np.array(all_preds); all_labels=np.array(all_labels)\noverall_acc = (all_preds==all_labels).mean()\nprint(f\"\\nOverall accuracy: {overall_acc:.3f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-20T12:27:06.474379Z","iopub.execute_input":"2025-12-20T12:27:06.474748Z"}},"outputs":[],"execution_count":null}]}