{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:37:43.750177Z","iopub.execute_input":"2026-02-02T17:37:43.750904Z","iopub.status.idle":"2026-02-02T17:37:44.058666Z","shell.execute_reply.started":"2026-02-02T17:37:43.750877Z","shell.execute_reply":"2026-02-02T17:37:44.05796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACCELEROMETER-ONLY TRANSFORMER\n# Single-file Kaggle version with analysis & plots\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import defaultdict\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"  # <<< CHANGE\nBATCH_SIZE = 32\nSEQ_LEN = 150\nEPOCHS = 10\nLR = 3e-4\nTOP_K = 3\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device:\", DEVICE)\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass AccDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n\n    def __init__(self, csv_path, seq_len=150):\n        self.df = pd.read_csv(csv_path)\n        self.seq_len = seq_len\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # Group by movement (= sequence)\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        print(f\"Loaded {len(self.sequences)} movements\")\n\n    def __len__(self):\n        return len(self.sequences)\n\n    def __getitem__(self, idx):\n        _, seq = self.sequences[idx]\n\n        acc = seq[self.ACC_COLS].values.astype(np.float32)\n\n        # Normalize per sequence\n        acc = (acc - acc.mean(axis=0)) / (acc.std(axis=0) + 1e-6)\n\n        L = min(len(acc), self.seq_len)\n        acc = acc[:L]\n\n        # Pad\n        pad = np.zeros((self.seq_len, 3), dtype=np.float32)\n        pad[:L] = acc\n\n        mask = np.zeros(self.seq_len, dtype=np.bool_)\n        mask[:L] = 1\n\n        label = seq[\"gesture_id\"].mode().iloc[0]\n\n        return {\n            \"acc\": torch.tensor(pad),\n            \"mask\": torch.tensor(mask),\n            \"label\": torch.tensor(label)\n        }\n\n# ============================================================\n# MODEL\n# ============================================================\n\nclass AccTransformer(nn.Module):\n    def __init__(self, num_classes, d_model=64, nhead=4, layers=4):\n        super().__init__()\n\n        self.input_proj = nn.Linear(3, d_model)\n        self.pos_emb = nn.Parameter(torch.randn(1, SEQ_LEN, d_model))\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            batch_first=True\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, layers)\n\n        self.cls_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, num_classes)\n        )\n\n    def forward(self, x, mask):\n        x = self.input_proj(x) + self.pos_emb\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        x = x.mean(dim=1)\n        return self.cls_head(x)\n\n# ============================================================\n# LOAD DATA\n# ============================================================\n\ndataset = AccDataset(CSV_PATH, SEQ_LEN)\nnum_classes = len(dataset.gesture_classes)\n\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_ds, test_ds = random_split(dataset, [train_size, test_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE)\n\n# ============================================================\n# TRAIN\n# ============================================================\n\nmodel = AccTransformer(num_classes).to(DEVICE)\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total_loss = 0\n\n    for batch in train_loader:\n        acc = batch[\"acc\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        label = batch[\"label\"].to(DEVICE)\n\n        optimizer.zero_grad()\n        logits = model(acc, mask)\n        loss = criterion(logits, label)\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n\n    print(f\"Epoch {epoch+1} | Loss: {total_loss / len(train_loader):.4f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\nmodel.eval()\nall_preds, all_labels = [], []\n\nwith torch.no_grad():\n    for batch in test_loader:\n        acc = batch[\"acc\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n\n        logits = model(acc, mask)\n        preds = logits.argmax(dim=1)\n\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(batch[\"label\"].numpy())\n\nall_preds = np.array(all_preds)\nall_labels = np.array(all_labels)\n\n# ============================================================\n# METRICS\n# ============================================================\n\noverall_acc = (all_preds == all_labels).mean()\nprint(f\"\\nOverall accuracy (accelerometer only): {overall_acc:.3f}\")\n\n# ---- Per-gesture accuracy\ngesture_acc = {}\ngesture_counts = {}\n\nfor g in range(num_classes):\n    idx = all_labels == g\n    if idx.sum() == 0:\n        continue\n    gesture_acc[g] = (all_preds[idx] == g).mean()\n    gesture_counts[g] = idx.sum()\n\n# ---- Ranking\nranking = sorted(\n    gesture_acc.items(),\n    key=lambda x: x[1],\n    reverse=True\n)\n\nprint(\"\\nGesture ranking (accelerometer detectability):\")\nprint(\"-\" * 50)\nfor g, acc in ranking:\n    print(f\"{dataset.gesture_classes[g]:20s} | Acc: {acc:.3f} | N={gesture_counts[g]}\")\n\n# ============================================================\n# CONFUSION MATRIX\n# ============================================================\n\nconf = np.zeros((num_classes, num_classes), dtype=int)\nfor t, p in zip(all_labels, all_preds):\n    conf[t, p] += 1\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(\n    conf,\n    xticklabels=dataset.gesture_classes,\n    yticklabels=dataset.gesture_classes,\n    cmap=\"Blues\",\n    fmt=\"d\"\n)\nplt.title(\"Confusion Matrix (Accelerometer Only)\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.tight_layout()\nplt.show()\n\n# ============================================================\n# TOP-K ACCURACY\n# ============================================================\n\ncorrect_topk = 0\n\nwith torch.no_grad():\n    for batch in test_loader:\n        acc = batch[\"acc\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        label = batch[\"label\"].to(DEVICE)\n\n        logits = model(acc, mask)\n        topk = logits.topk(TOP_K, dim=1).indices\n\n        for i in range(label.size(0)):\n            if label[i] in topk[i]:\n                correct_topk += 1\n\ntopk_acc = correct_topk / len(all_labels)\nprint(f\"\\nTop-{TOP_K} accuracy: {topk_acc:.3f}\")\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-02T17:37:44.060051Z","iopub.execute_input":"2026-02-02T17:37:44.060427Z","iopub.status.idle":"2026-02-02T17:40:07.185172Z","shell.execute_reply.started":"2026-02-02T17:37:44.060405Z","shell.execute_reply":"2026-02-02T17:40:07.184438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ROTATION-ONLY TRANSFORMER\n# Single-file Kaggle version with analysis & plots\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nBATCH_SIZE = 32\nSEQ_LEN = 150\nEPOCHS = 10\nLR = 1e-4  # Reduced from 3e-4\nTOP_K = 3\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device:\", DEVICE)\n\n# Set seed for reproducibility\ntorch.manual_seed(42)\nnp.random.seed(42)\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass RotDataset(Dataset):\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n\n    def __init__(self, csv_path, seq_len=150):\n        self.df = pd.read_csv(csv_path)\n        self.seq_len = seq_len\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # Group by movement (= sequence)\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        print(f\"Loaded {len(self.sequences)} movements\")\n        print(f\"Number of gesture classes: {len(self.gesture_classes)}\")\n        print(f\"Gesture classes: {self.gesture_classes}\")\n\n    def __len__(self):\n        return len(self.sequences)\n\n    def __getitem__(self, idx):\n        _, seq = self.sequences[idx]\n\n        rot = seq[self.ROT_COLS].values.astype(np.float32)\n        \n        # Check for NaN values and replace with zeros\n        if np.any(np.isnan(rot)):\n            rot = np.nan_to_num(rot)\n\n        # Normalize quaternions to unit length per row\n        norms = np.linalg.norm(rot, axis=1, keepdims=True)\n        # Handle zero norms - set to 1 to avoid division by zero\n        norms[norms == 0] = 1.0\n        rot = rot / norms\n\n        L = min(len(rot), self.seq_len)\n        rot = rot[:L]\n\n        # Pad\n        pad = np.zeros((self.seq_len, 4), dtype=np.float32)\n        pad[:L] = rot\n\n        # Create mask: 1 for real data, 0 for padding\n        mask = torch.zeros(self.seq_len, dtype=torch.bool)\n        mask[:L] = 1\n\n        label = seq[\"gesture_id\"].mode().iloc[0]\n\n        return {\n            \"rot\": torch.tensor(pad, dtype=torch.float32),\n            \"mask\": mask,\n            \"label\": torch.tensor(label, dtype=torch.long)\n        }\n\n# ============================================================\n# MODEL with improved stability\n# ============================================================\n\nclass RotTransformer(nn.Module):\n    def __init__(self, num_classes, d_model=64, nhead=4, layers=4, dropout=0.1):\n        super().__init__()\n        \n        self.d_model = d_model\n        \n        # Input projection with better initialization\n        self.input_proj = nn.Linear(4, d_model)\n        nn.init.xavier_uniform_(self.input_proj.weight)\n        nn.init.zeros_(self.input_proj.bias)\n        \n        # Positional embedding with smaller initialization\n        self.pos_emb = nn.Parameter(torch.randn(1, SEQ_LEN, d_model) * 0.01)\n        \n        # Add layer normalization before transformer\n        self.ln1 = nn.LayerNorm(d_model)\n        \n        # Transformer encoder with dropout\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=d_model * 4,\n            dropout=dropout,\n            batch_first=True,\n            activation='gelu'  # More stable than relu\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, layers)\n        \n        # Pooling layer\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Classification head with more layers and better normalization\n        self.cls_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(d_model, num_classes)\n        )\n        \n        # Initialize classification head properly\n        for layer in self.cls_head:\n            if isinstance(layer, nn.Linear):\n                nn.init.xavier_uniform_(layer.weight)\n                nn.init.zeros_(layer.bias)\n\n    def forward(self, x, mask):\n        # Input projection\n        x = self.input_proj(x)\n        \n        # Add positional encoding\n        x = x + self.pos_emb\n        \n        # Layer norm\n        x = self.ln1(x)\n        \n        # Pass through transformer\n        # Note: src_key_padding_mask expects True for padded positions\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        \n        # Mean pooling over sequence dimension (considering only non-padded positions)\n        # Create attention mask for pooling\n        mask_expanded = mask.unsqueeze(-1).float()  # (batch, seq_len, 1)\n        x_masked = x * mask_expanded\n        sum_pool = torch.sum(x_masked, dim=1)\n        count = mask.sum(dim=1, keepdim=True).clamp(min=1)\n        x = sum_pool / count\n        \n        # Classification\n        return self.cls_head(x)\n\n# ============================================================\n# LOAD DATA\n# ============================================================\n\ndataset = RotDataset(CSV_PATH, SEQ_LEN)\nnum_classes = len(dataset.gesture_classes)\nprint(f\"\\nNumber of classes: {num_classes}\")\n\n# Check class distribution\nclass_counts = dataset.df.groupby(\"gesture_id\").size()\nprint(\"\\nClass distribution:\")\nfor gid, count in class_counts.items():\n    print(f\"  Class {gid} ({dataset.gesture_classes[gid]}): {count} samples\")\n\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_ds, test_ds = random_split(dataset, [train_size, test_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE)\n\n# ============================================================\n# TRAINING with gradient clipping and learning rate warmup\n# ============================================================\n\nmodel = RotTransformer(num_classes, dropout=0.1).to(DEVICE)\n\n# Use AdamW with weight decay\noptimizer = torch.optim.AdamW(\n    model.parameters(), \n    lr=LR,\n    weight_decay=1e-4,\n    betas=(0.9, 0.999)\n)\n\n# Learning rate scheduler\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n    optimizer, \n    T_max=EPOCHS * len(train_loader),\n    eta_min=LR * 0.01\n)\n\ncriterion = nn.CrossEntropyLoss()\n\n# Track training metrics\ntrain_losses = []\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total_loss = 0\n    correct = 0\n    total = 0\n    \n    for batch_idx, batch in enumerate(train_loader):\n        rot = batch[\"rot\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        label = batch[\"label\"].to(DEVICE)\n        \n        optimizer.zero_grad()\n        logits = model(rot, mask)\n        \n        # Add label smoothing for stability\n        loss = criterion(logits, label)\n        \n        # Check for NaN loss\n        if torch.isnan(loss):\n            print(f\"WARNING: NaN loss at batch {batch_idx}\")\n            continue\n            \n        loss.backward()\n        \n        # Gradient clipping\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        \n        optimizer.step()\n        scheduler.step()\n        \n        total_loss += loss.item()\n        \n        # Calculate accuracy\n        preds = logits.argmax(dim=1)\n        correct += (preds == label).sum().item()\n        total += label.size(0)\n        \n        # Print progress\n        if batch_idx % 20 == 0:\n            current_lr = scheduler.get_last_lr()[0]\n            print(f\"Epoch {epoch+1} | Batch {batch_idx} | Loss: {loss.item():.4f} | LR: {current_lr:.6f}\")\n    \n    avg_loss = total_loss / len(train_loader)\n    train_acc = correct / total\n    train_losses.append(avg_loss)\n    \n    print(f\"Epoch {epoch+1} | Avg Loss: {avg_loss:.4f} | Train Acc: {train_acc:.4f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\nmodel.eval()\nall_preds, all_labels = [], []\nall_probs = []\n\nwith torch.no_grad():\n    for batch in test_loader:\n        rot = batch[\"rot\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        labels = batch[\"label\"].numpy()\n\n        logits = model(rot, mask)\n        probs = F.softmax(logits, dim=1)\n        preds = logits.argmax(dim=1)\n\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(labels)\n        all_probs.extend(probs.cpu().numpy())\n\nall_preds = np.array(all_preds)\nall_labels = np.array(all_labels)\nall_probs = np.array(all_probs)\n\n# ============================================================\n# METRICS\n# ============================================================\n\noverall_acc = (all_preds == all_labels).mean()\nprint(f\"\\nOverall accuracy (rotation only): {overall_acc:.3f}\")\n\n# ---- Per-gesture accuracy\ngesture_acc = {}\ngesture_counts = {}\n\nfor g in range(num_classes):\n    idx = all_labels == g\n    if idx.sum() == 0:\n        continue\n    gesture_acc[g] = (all_preds[idx] == g).mean()\n    gesture_counts[g] = idx.sum()\n\n# ---- Ranking\nranking = sorted(\n    gesture_acc.items(),\n    key=lambda x: x[1],\n    reverse=True\n)\n\nprint(\"\\nGesture ranking (rotation detectability):\")\nprint(\"-\" * 50)\nfor g, acc in ranking:\n    print(f\"{dataset.gesture_classes[g]:20s} | Acc: {acc:.3f} | N={gesture_counts[g]}\")\n\n# ============================================================\n# CONFUSION MATRIX\n# ============================================================\n\nconf = np.zeros((num_classes, num_classes), dtype=int)\nfor t, p in zip(all_labels, all_preds):\n    conf[t, p] += 1\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(\n    conf,\n    xticklabels=dataset.gesture_classes,\n    yticklabels=dataset.gesture_classes,\n    cmap=\"Blues\",\n    fmt=\"d\",\n    annot=True,\n    annot_kws={'size': 8}\n)\nplt.title(f\"Confusion Matrix (Rotation Only) - Overall Acc: {overall_acc:.3f}\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.tight_layout()\nplt.savefig(\"confusion_matrix.png\", dpi=150)\nplt.show()\n\n# ============================================================\n# TOP-K ACCURACY\n# ============================================================\n\ncorrect_topk = 0\n\nfor i in range(len(all_labels)):\n    label = all_labels[i]\n    prob = all_probs[i]\n    top_k_indices = np.argsort(prob)[-TOP_K:][::-1]\n    if label in top_k_indices:\n        correct_topk += 1\n\ntopk_acc = correct_topk / len(all_labels)\nprint(f\"\\nTop-{TOP_K} accuracy: {topk_acc:.3f}\")\n\n# ============================================================\n# ADDITIONAL DIAGNOSTICS\n# ============================================================\n\n# Plot training loss\nplt.figure(figsize=(10, 4))\nplt.plot(train_losses, label='Training Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training Loss Curve')\nplt.legend()\nplt.grid(True, alpha=0.3)\nplt.tight_layout()\nplt.savefig(\"training_loss.png\", dpi=150)\nplt.show()\n\n# Calculate per-class metrics\nfrom sklearn.metrics import classification_report\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"Classification Report:\")\nprint(\"=\"*50)\nprint(classification_report(\n    all_labels, \n    all_preds, \n    target_names=dataset.gesture_classes,\n    digits=3\n))","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-02T17:40:07.186133Z","iopub.execute_input":"2026-02-02T17:40:07.18662Z","iopub.status.idle":"2026-02-02T17:41:45.580766Z","shell.execute_reply.started":"2026-02-02T17:40:07.186599Z","shell.execute_reply":"2026-02-02T17:41:45.579911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACCELEROMETER-ONLY TRANSFORMER\n# Single-file Kaggle version with analysis & plots\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import defaultdict\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"  # <<< CHANGE\nBATCH_SIZE = 32\nSEQ_LEN = 21\nEPOCHS = 10\nLR = 3e-4\nTOP_K = 3\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device:\", DEVICE)\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass RotDataset(Dataset):\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n\n    def __init__(self, csv_path, seq_len=150):\n        self.df = pd.read_csv(csv_path)\n        self.seq_len = seq_len\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # Group by movement (= sequence)\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        print(f\"Loaded {len(self.sequences)} movements\")\n        print(f\"Number of gesture classes: {len(self.gesture_classes)}\")\n        print(f\"Gesture classes: {self.gesture_classes}\")\n\n    def __len__(self):\n        return len(self.sequences)\n\n    def __getitem__(self, idx):\n        _, seq = self.sequences[idx]\n\n        rot = seq[self.ROT_COLS].values.astype(np.float32)\n        \n        # Check for NaN values and replace with zeros\n        if np.any(np.isnan(rot)):\n            rot = np.nan_to_num(rot)\n\n        # Normalize quaternions to unit length per row\n        norms = np.linalg.norm(rot, axis=1, keepdims=True)\n        # Handle zero norms - set to 1 to avoid division by zero\n        norms[norms == 0] = 1.0\n        rot = rot / norms\n\n        L = min(len(rot), self.seq_len)\n        rot = rot[:L]\n\n        # Pad\n        pad = np.zeros((self.seq_len, 4), dtype=np.float32)\n        pad[:L] = rot\n\n        # Create mask: 1 for real data, 0 for padding\n        mask = torch.zeros(self.seq_len, dtype=torch.bool)\n        mask[:L] = 1\n\n        label = seq[\"gesture_id\"].mode().iloc[0]\n\n        return {\n            \"rot\": torch.tensor(pad, dtype=torch.float32),\n            \"mask\": mask,\n            \"label\": torch.tensor(label, dtype=torch.long)\n        }\n\n# ============================================================\n# MODEL with improved stability\n# ============================================================\n\nclass RotTransformer(nn.Module):\n    def __init__(self, num_classes, d_model=64, nhead=4, layers=4, dropout=0.1):\n        super().__init__()\n        \n        self.d_model = d_model\n        \n        # Input projection with better initialization\n        self.input_proj = nn.Linear(4, d_model)\n        nn.init.xavier_uniform_(self.input_proj.weight)\n        nn.init.zeros_(self.input_proj.bias)\n        \n        # Positional embedding with smaller initialization\n        self.pos_emb = nn.Parameter(torch.randn(1, SEQ_LEN, d_model) * 0.01)\n        \n        # Add layer normalization before transformer\n        self.ln1 = nn.LayerNorm(d_model)\n        \n        # Transformer encoder with dropout\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=d_model * 4,\n            dropout=dropout,\n            batch_first=True,\n            activation='gelu'  # More stable than relu\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, layers)\n        \n        # Pooling layer\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Classification head with more layers and better normalization\n        self.cls_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(d_model, num_classes)\n        )\n        \n        # Initialize classification head properly\n        for layer in self.cls_head:\n            if isinstance(layer, nn.Linear):\n                nn.init.xavier_uniform_(layer.weight)\n                nn.init.zeros_(layer.bias)\n\n    def forward(self, x, mask):\n        # Input projection\n        x = self.input_proj(x)\n        \n        # Add positional encoding\n        x = x + self.pos_emb\n        \n        # Layer norm\n        x = self.ln1(x)\n        \n        # Pass through transformer\n        # Note: src_key_padding_mask expects True for padded positions\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        \n        # Mean pooling over sequence dimension (considering only non-padded positions)\n        # Create attention mask for pooling\n        mask_expanded = mask.unsqueeze(-1).float()  # (batch, seq_len, 1)\n        x_masked = x * mask_expanded\n        sum_pool = torch.sum(x_masked, dim=1)\n        count = mask.sum(dim=1, keepdim=True).clamp(min=1)\n        x = sum_pool / count\n        \n        # Classification\n        return self.cls_head(x)\n# ============================================================\nclass AccDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n\n    def __init__(self, csv_path):\n        self.df = pd.read_csv(csv_path)\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # Group by movement (= sequence)\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        print(f\"Loaded {len(self.sequences)} movements\")\n\n    def __len__(self):\n        return len(self.sequences)\n\n    def __getitem__(self, idx):\n        _, seq = self.sequences[idx]\n\n        acc = seq[self.ACC_COLS].values.astype(np.float32)\n\n        # Normalize per sequence\n        acc = (acc - acc.mean(axis=0)) / (acc.std(axis=0) + 1e-6)\n\n        label = seq[\"gesture_id\"].mode().iloc[0]\n\n        return {\"acc\": torch.tensor(acc), \"length\": len(acc), \"label\": torch.tensor(label)}\n\n\n# ============================================================\n# COLLATE FUNCTION FOR DYNAMIC PADDING\n# ============================================================\n\ndef collate_acc_fn(batch):\n    batch_size = len(batch)\n    lengths = [b[\"length\"] for b in batch]\n    max_len = max(lengths)\n\n    accs = torch.zeros(batch_size, max_len, 3, dtype=torch.float32)\n    masks = torch.zeros(batch_size, max_len, dtype=torch.bool)\n    labels = torch.zeros(batch_size, dtype=torch.long)\n\n    for i, b in enumerate(batch):\n        L = b[\"length\"]\n        accs[i, :L] = b[\"acc\"]\n        masks[i, :L] = 1\n        labels[i] = b[\"label\"]\n\n    return {\"acc\": accs, \"mask\": masks, \"label\": labels}\n\n\n# ============================================================\n# TRANSFORMER MODEL\n# ============================================================\n\nclass AccTransformer(nn.Module):\n    def __init__(self, num_classes, d_model=64, nhead=4, layers=4):\n        super().__init__()\n        self.input_proj = nn.Linear(3, d_model)\n        self.encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            batch_first=True\n        )\n        self.encoder = nn.TransformerEncoder(self.encoder_layer, layers)\n        self.cls_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, num_classes)\n        )\n\n    def forward(self, x, mask):\n        # x: (batch, seq_len, 3)\n        # mask: (batch, seq_len)\n        batch_size, seq_len, _ = x.size()\n\n        # Dynamic positional embeddings\n        pos_emb = torch.randn(1, seq_len, self.input_proj.out_features, device=x.device)\n        x = self.input_proj(x) + pos_emb\n\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        x = x.mean(dim=1)\n        return self.cls_head(x)\n    \nclass AccWindowDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n\n    def __init__(self, csv_path, window_before=10, window_after=10):\n        self.df = pd.read_csv(csv_path)\n        self.window_before = window_before\n        self.window_after = window_after\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # Group by movement (= sequence)\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n\n        self.windows = []\n        for _, seq in self.sequences:\n            gesture_rows = seq[seq[\"phase\"] == \"Gesture\"]\n            if gesture_rows.empty:\n                center_pos = 0  # fallback\n            else:\n                gesture_pos = gesture_rows.index[0]\n                center_pos = seq.index.get_loc(gesture_pos)           # convert to position 0..len-1\n\n            start = max(0, center_pos - WINDOW_BEFORE)\n            end   = min(len(seq), center_pos + WINDOW_AFTER)\n\n            window_data = seq.iloc[start:end][self.ACC_COLS].values.astype(np.float32)\n            if window_data.shape[0] == 0:\n                continue  # skip empty windows\n\n            label = seq[\"gesture_id\"].mode().iloc[0]\n            self.windows.append((window_data, label))\n\n        print(f\"Created {len(self.windows)} windows\")\n\n    def __len__(self):\n        return len(self.windows)\n\n    def __getitem__(self, idx):\n        data, label = self.windows[idx]\n        L = data.shape[0]\n        # Pad if needed\n        pad_len = self.window_before + self.window_after + 1\n        if L < pad_len:\n            pad = np.zeros((pad_len, 3), dtype=np.float32)\n            pad[:L] = data\n            mask = np.zeros(pad_len, dtype=np.bool_)\n            mask[:L] = 1\n            data = pad\n        else:\n            mask = np.ones(L, dtype=np.bool_)\n        return {\n            \"acc\": torch.tensor(data, dtype=torch.float32),\n            \"mask\": torch.tensor(mask),\n            \"label\": torch.tensor(label)\n        }\n\n# ============================================================\n# MODEL\n# ============================================================\n\nclass AccWindowTransformer(nn.Module):\n    def __init__(self, num_classes, window_len, d_model=64, nhead=4, layers=4):\n        super().__init__()\n        self.input_proj = nn.Linear(3, d_model)\n        self.pos_emb = nn.Parameter(torch.randn(1, window_len, d_model))\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            batch_first=True\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, layers)\n\n        self.cls_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, num_classes)\n        )\n\n    def forward(self, x, mask):\n        x = self.input_proj(x) + self.pos_emb\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        x = x.mean(dim=1)\n        return self.cls_head(x)\n\nclass RotWindowDataset(Dataset):\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n\n    def __init__(self, csv_path, window_before=10, window_after=10):\n        self.df = pd.read_csv(csv_path)\n        self.window_before = window_before\n        self.window_after = window_after\n        self.window_len = window_before + window_after + 1\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        # Group by movement (= sequence)\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n\n        self.windows = []\n        for _, seq in self.sequences:\n            seq = seq.reset_index(drop=True)\n\n            gesture_rows = seq[seq[\"phase\"] == \"Gesture\"]\n            if gesture_rows.empty:\n                continue  # skip invalid sequences\n\n            center_pos = gesture_rows.index[0]\n\n            start = center_pos - self.window_before\n            end = center_pos + self.window_after + 1\n\n            start_clamped = max(0, start)\n            end_clamped = min(len(seq), end)\n\n            rot = seq.iloc[start_clamped:end_clamped][self.ROT_COLS].values.astype(np.float32)\n\n            if rot.shape[0] == 0:\n                continue\n\n            # ---- Handle NaNs\n            rot = np.nan_to_num(rot)\n\n            # ---- Normalize quaternions (row-wise)\n            norms = np.linalg.norm(rot, axis=1, keepdims=True)\n            norms[norms < 1e-6] = 1.0\n            rot = rot / norms\n\n            # ---- Pad to fixed window\n            padded = np.zeros((self.window_len, 4), dtype=np.float32)\n            mask = np.zeros(self.window_len, dtype=np.bool_)\n\n            insert_pos = start_clamped - start\n            padded[insert_pos:insert_pos + len(rot)] = rot\n            mask[insert_pos:insert_pos + len(rot)] = True\n\n            label = seq[\"gesture_id\"].mode().iloc[0]\n\n            self.windows.append((padded, mask, label))\n\n        print(f\"Created {len(self.windows)} rotation windows\")\n\n    def __len__(self):\n        return len(self.windows)\n\n    def __getitem__(self, idx):\n        rot, mask, label = self.windows[idx]\n        return {\n            \"rot\": torch.tensor(rot, dtype=torch.float32),\n            \"mask\": torch.tensor(mask, dtype=torch.bool),\n            \"label\": torch.tensor(label, dtype=torch.long),\n        }\n\nclass RotationWinTransformer(nn.Module):\n    def __init__(self, num_classes, window_len, d_model=64, nhead=4, layers=4, dropout=0.1):\n        super().__init__()\n\n        self.input_proj = nn.Linear(4, d_model)\n        nn.init.xavier_uniform_(self.input_proj.weight)\n        nn.init.zeros_(self.input_proj.bias)\n\n        self.pos_emb = nn.Parameter(torch.randn(1, window_len, d_model) * 0.01)\n        self.ln1 = nn.LayerNorm(d_model)\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=d_model * 4,\n            dropout=dropout,\n            batch_first=True,\n            activation=\"gelu\",\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, layers)\n\n        self.cls_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(d_model, num_classes),\n        )\n\n        for m in self.cls_head:\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n                nn.init.zeros_(m.bias)\n\n    def forward(self, x, mask):\n        \"\"\"\n        x: (B, T, 4)\n        mask: (B, T)\n        \"\"\"\n\n        x = self.input_proj(x)\n        x = x + self.pos_emb\n        x = self.ln1(x)\n\n        x = self.encoder(x, src_key_padding_mask=~mask)\n\n        # ---- Masked mean pooling\n        mask_f = mask.unsqueeze(-1).float()\n        x = (x * mask_f).sum(dim=1) / mask_f.sum(dim=1).clamp(min=1.0)\n\n        return self.cls_head(x)\n# ============================================================\n# LOAD DATA\n# ============================================================\n\nBATCH_SIZE = 32\nWINDOW_BEFORE = 10\nWINDOW_AFTER = 10\nD_MODEL = 64\nNHEAD = 4\nLAYERS = 4\nEPOCHS = 10\nLR = 3e-4\nTOP_K = 3\n# dataset = AccWindowDataset(CSV_PATH, WINDOW_BEFORE, WINDOW_AFTER)\ndataset = RotWindowDataset(CSV_PATH, WINDOW_BEFORE, WINDOW_AFTER)\n# dataset = AccDataset(CSV_PATH)\n# dataset = RotDataset(CSV_PATH, SEQ_LEN)\nnum_classes = len(dataset.gesture_classes)\n\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_ds, test_ds = random_split(dataset, [train_size, test_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE)\n\n# ============================================================\n# TRAIN\n# ============================================================\n\nmodel = RotationWinTransformer(num_classes,21).to(DEVICE)\n# model = AccWindowTransformer(num_classes,21, D_MODEL, NHEAD, LAYERS).to(DEVICE)\n# model = AccTransformer(num_classes).to(DEVICE)\n# model = RotTransformer(num_classes).to(DEVICE)\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total_loss = 0\n\n    for batch in train_loader:\n        # acc = batch[\"acc\"].to(DEVICE)\n        rot = batch[\"rot\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        label = batch[\"label\"].to(DEVICE)\n\n        optimizer.zero_grad()\n        logits = model(rot, mask)\n        # logits = model(acc, mask)\n        loss = criterion(logits, label)\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n\n    print(f\"Epoch {epoch+1} | Loss: {total_loss / len(train_loader):.4f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\nmodel.eval()\nall_preds, all_labels = [], []\n\nwith torch.no_grad():\n    for batch in test_loader:\n        rot = batch[\"rot\"].to(DEVICE)\n        # acc = batch[\"acc\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n\n        logits = model(rot, mask)\n        # logits = model(acc, mask)\n        preds = logits.argmax(dim=1)\n\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(batch[\"label\"].numpy())\n\nall_preds = np.array(all_preds)\nall_labels = np.array(all_labels)\n\n# ============================================================\n# METRICS\n# ============================================================\n\noverall_acc = (all_preds == all_labels).mean()\nprint(f\"\\nOverall accuracy (rotation only): {overall_acc:.3f}\")\n# print(f\"\\nOverall accuracy (rotation only): {overall_acc:.3f}\")\n\n# ---- Per-gesture accuracy\ngesture_acc = {}\ngesture_counts = {}\n\nfor g in range(num_classes):\n    idx = all_labels == g\n    if idx.sum() == 0:\n        continue\n    gesture_acc[g] = (all_preds[idx] == g).mean()\n    gesture_counts[g] = idx.sum()\n\n# ---- Ranking\nranking = sorted(\n    gesture_acc.items(),\n    key=lambda x: x[1],\n    reverse=True\n)\n\nprint(\"\\nGesture ranking (rotation detectability):\")\n# print(\"\\nGesture ranking (accelerometer detectability):\")\nprint(\"-\" * 50)\nfor g, acc in ranking:\n    print(f\"{dataset.gesture_classes[g]:20s} | Acc: {acc:.3f} | N={gesture_counts[g]}\")\n\n# ============================================================\n# CONFUSION MATRIX\n# ============================================================\n\nconf = np.zeros((num_classes, num_classes), dtype=int)\nfor t, p in zip(all_labels, all_preds):\n    conf[t, p] += 1\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(\n    conf,\n    xticklabels=dataset.gesture_classes,\n    yticklabels=dataset.gesture_classes,\n    cmap=\"Blues\",\n    fmt=\"d\"\n)\n# plt.title(\"Confusion Matrix (Accelerometer Only)\")\nplt.title(\"Confusion Matrix (Rotation Only)\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.tight_layout()\nplt.show()\n\n# ============================================================\n# TOP-K ACCURACY\n# ============================================================\n\ncorrect_topk = 0\n\nwith torch.no_grad():\n    for batch in test_loader:\n        rot = batch[\"rot\"].to(DEVICE)\n        # acc = batch[\"acc\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        label = batch[\"label\"].to(DEVICE)\n\n        logits = model(rot, mask)\n        # logits = model(acc, mask)\n        topk = logits.topk(TOP_K, dim=1).indices\n\n        for i in range(label.size(0)):\n            if label[i] in topk[i]:\n                correct_topk += 1\n\ntopk_acc = correct_topk / len(all_labels)\nprint(f\"\\nTop-{TOP_K} accuracy: {topk_acc:.3f}\")\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-02T17:41:45.582383Z","iopub.execute_input":"2026-02-02T17:41:45.582618Z","iopub.status.idle":"2026-02-02T17:42:40.529939Z","shell.execute_reply.started":"2026-02-02T17:41:45.582599Z","shell.execute_reply":"2026-02-02T17:42:40.529166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# IMPROVED FUSED TRANSFORMER FOR SUBTLE GESTURE RECOGNITION\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, random_split, WeightedRandomSampler\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import defaultdict\nfrom scipy.spatial.transform import Rotation as R\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nBATCH_SIZE = 64\nSEQ_LEN = 21  # 10 + 10 + 1\nEPOCHS = 30\nLR = 2e-4\nTOP_K = 3\nD_MODEL = 128  # Increased capacity\nNHEAD = 8\nNLAYERS = 4\nDROPOUT = 0.3\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device:\", DEVICE)\n\n# Set seeds for reproducibility\ntorch.manual_seed(42)\nnp.random.seed(42)\n\n# ============================================================\n# ENHANCED DATA HELPERS WITH MORE FEATURES\n# ============================================================\n\ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    rot_cleaned = rot_data.copy()\n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        if missing_count == 0:\n            norm = np.linalg.norm(row)\n            rot_cleaned[i] = row / norm if norm > 1e-8 else [1.0, 0.0, 0.0, 0.0]\n        elif missing_count == 1:\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            sum_squares = np.sum(valid_values**2)\n            missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n            if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n                if rot_cleaned[i-1, missing_idx] < 0:\n                    missing_value = -missing_value\n            rot_cleaned[i, missing_idx] = missing_value\n            rot_cleaned[i, ~np.isnan(row)] = valid_values\n        else:\n            rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n    return rot_cleaned\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    \"\"\"Convert acceleration from device to world coordinates\"\"\"\n    try:\n        rot_scipy = rot[:, [1, 2, 3, 0]]  # [w,x,y,z] -> [x,y,z,w] for scipy\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        if np.any(norms < 1e-8):\n            mask = norms < 1e-8\n            rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0]\n        r = R.from_quat(rot_scipy)\n        acc_world = r.apply(acc)\n        return acc_world\n    except Exception as e:\n        print(f\"Warning: World coordinate transformation failed: {e}\")\n        return acc.copy()\n\ndef compute_angular_velocity(rot_data, time_delta=1/200):\n    \"\"\"Compute angular velocity from quaternions\"\"\"\n    num_samples = rot_data.shape[0]\n    angular_vel = np.zeros((num_samples, 3))\n    for i in range(num_samples - 1):\n        try:\n            r1 = R.from_quat(rot_data[i])\n            r2 = R.from_quat(rot_data[i+1])\n            delta = r1.inv() * r2\n            angular_vel[i] = delta.as_rotvec() / time_delta\n        except:\n            continue\n    return angular_vel\n\ndef compute_motion_intensity_features(acc, rot):\n    \"\"\"Compute features that help distinguish subtle vs large motions\"\"\"\n    # Magnitude features\n    acc_mag = np.sqrt(np.sum(acc**2, axis=1))\n    acc_mag_smooth = np.convolve(acc_mag, np.ones(5)/5, mode='same')\n    \n    # Jerk (rate of change of acceleration)\n    jerk = np.zeros_like(acc)\n    if len(acc) > 1:\n        jerk[1:] = np.diff(acc, axis=0) * 200  # 200Hz sampling\n    jerk_mag = np.sqrt(np.sum(jerk**2, axis=1))\n    \n    # Angular features\n    angular_vel = compute_angular_velocity(rot)\n    angular_vel_mag = np.sqrt(np.sum(angular_vel**2, axis=1))\n    \n    # Motion variability (subtle motions are more periodic)\n    motion_variability = np.zeros(len(acc))\n    if len(acc) > 10:\n        # Compute rolling standard deviation to detect repetitive motions\n        window = min(10, len(acc))\n        for i in range(len(acc)):\n            start = max(0, i - window//2)\n            end = min(len(acc), i + window//2)\n            if end > start:\n                motion_variability[i] = np.std(acc[start:end])\n    \n    return {\n        'acc_mag': acc_mag,\n        'acc_mag_smooth': acc_mag_smooth,\n        'jerk_mag': jerk_mag,\n        'angular_vel_mag': angular_vel_mag,\n        'motion_variability': motion_variability\n    }\n\ndef compute_temporal_features(data, window_size=5):\n    \"\"\"Compute temporal features (moving average) for each column of data\"\"\"\n    n_samples = len(data)\n    n_features = data.shape[1]\n    features = np.zeros((n_samples, n_features))  # FIXED: Use n_features instead of 4\n    kernel = np.ones(window_size) / window_size\n    \n    for i in range(n_features):\n        features[:, i] = np.convolve(data[:, i], kernel, mode='same')\n    \n    return features\n\n# ============================================================\n# ENHANCED DATASET WITH CLASS-SPECIFIC PROCESSING\n# ============================================================\n\nclass EnhancedMultimodalDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n    \n    def __init__(self, csv_path, window_before=10, window_after=10, mode='train'):\n        self.df = pd.read_csv(csv_path)\n        self.window_before = window_before\n        self.window_after = window_after\n        self.window_len = window_before + window_after + 1\n        self.mode = mode\n        \n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n        \n        # Analyze gesture types\n        self.gesture_type = {}\n        for i, gesture in enumerate(self.gesture_classes):\n            gesture_lower = gesture.lower()\n            if any(word in gesture_lower for word in ['pinch', 'scratch', 'pull']):\n                self.gesture_type[i] = 'subtle'\n            elif any(word in gesture_lower for word in ['drink', 'feel', 'glasses', 'text', 'wave', 'write']):\n                self.gesture_type[i] = 'large'\n            else:\n                self.gesture_type[i] = 'other'\n        \n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        self.windows = []\n        \n        print(f\"Loaded {len(self.sequences)} sequences\")\n        subtle_count = sum(1 for v in self.gesture_type.values() if v == 'subtle')\n        large_count = sum(1 for v in self.gesture_type.values() if v == 'large')\n        print(f\"Gesture types: Subtle={subtle_count}, Large={large_count}\")\n\n    def __len__(self):\n        return len(self.windows)\n\n    def process_sequence(self, seq):\n        \"\"\"Process a single sequence with enhanced features\"\"\"\n        seq = seq.reset_index(drop=True)\n        \n        # Find gesture phase\n        gesture_rows = seq[seq[\"phase\"] == \"Gesture\"]\n        if gesture_rows.empty:\n            return None\n        \n        center = gesture_rows.index[0]\n        start = max(0, center - self.window_before)\n        end = min(len(seq), center + self.window_after + 1)\n        \n        # Extract raw data\n        acc_raw = seq.iloc[start:end][self.ACC_COLS].values.astype(np.float32)\n        rot_raw = seq.iloc[start:end][self.ROT_COLS].values.astype(np.float32)\n        \n        if len(acc_raw) == 0 or len(rot_raw) == 0:\n            return None\n        \n        # Handle missing values in rotation\n        rot_cleaned = handle_quaternion_missing_values(rot_raw)\n        \n        # Normalize quaternions\n        norms = np.linalg.norm(rot_cleaned, axis=1, keepdims=True)\n        norms[norms < 1e-6] = 1.0\n        rot_norm = rot_cleaned / norms\n        \n        # Compute world acceleration\n        acc_world = compute_world_acceleration(acc_raw, rot_norm)\n        \n        # Compute enhanced features\n        motion_features = compute_motion_intensity_features(acc_world, rot_norm)\n        \n        # Compute angular velocity\n        angular_vel = compute_angular_velocity(rot_norm)\n        \n        # Create feature-rich representations\n        # Accelerometer features: [world_x, world_y, world_z, magnitude, jerk_mag, variability]\n        acc_features = np.column_stack([\n            acc_world,\n            motion_features['acc_mag'].reshape(-1, 1),\n            motion_features['jerk_mag'].reshape(-1, 1),\n            motion_features['motion_variability'].reshape(-1, 1)\n        ])  # Shape: (n_samples, 6)\n        \n        # Rotation features: [quat_w, quat_x, quat_y, quat_z, angular_vel_x, angular_vel_y, angular_vel_z, angular_mag]\n        rot_features = np.column_stack([\n            rot_norm,\n            angular_vel,\n            motion_features['angular_vel_mag'].reshape(-1, 1)\n        ])  # Shape: (n_samples, 8)\n        \n        # Add temporal features\n        acc_temporal = compute_temporal_features(acc_features)\n        rot_temporal = compute_temporal_features(rot_features[:, :4])  # Only on quaternions\n        \n        acc_features = np.column_stack([acc_features, acc_temporal])  # Now (n_samples, 12)\n        rot_features = np.column_stack([rot_features, rot_temporal])  # Now (n_samples, 12)\n        \n        # Normalize per window\n        acc_mean = acc_features.mean(axis=0, keepdims=True)\n        acc_std = acc_features.std(axis=0, keepdims=True) + 1e-6\n        acc_features = (acc_features - acc_mean) / acc_std\n        \n        rot_mean = rot_features.mean(axis=0, keepdims=True)\n        rot_std = rot_features.std(axis=0, keepdims=True) + 1e-6\n        rot_features = (rot_features - rot_mean) / rot_std\n        \n        # Padding\n        actual_len = len(acc_features)\n        acc_pad = np.zeros((self.window_len, acc_features.shape[1]), dtype=np.float32)\n        rot_pad = np.zeros((self.window_len, rot_features.shape[1]), dtype=np.float32)\n        mask = np.zeros(self.window_len, dtype=np.bool_)\n        \n        insert_start = 0\n        acc_pad[insert_start:insert_start + actual_len] = acc_features\n        rot_pad[insert_start:insert_start + actual_len] = rot_features\n        mask[insert_start:insert_start + actual_len] = True\n        \n        label = seq[\"gesture_id\"].mode().iloc[0]\n        \n        # Store gesture type for weighted sampling\n        gesture_type = self.gesture_type.get(label, 'other')\n        \n        return acc_pad, rot_pad, mask, label, gesture_type\n\n    def build_windows(self):\n        \"\"\"Build all windows (can be called after initialization if needed)\"\"\"\n        self.windows = []\n        self.gesture_types = []\n        \n        for _, seq in self.sequences:\n            result = self.process_sequence(seq)\n            if result is not None:\n                acc_pad, rot_pad, mask, label, gtype = result\n                self.windows.append((acc_pad, rot_pad, mask, label))\n                self.gesture_types.append(gtype)\n        \n        print(f\"Created {len(self.windows)} windows\")\n        if self.windows:\n            print(f\"Accelerometer features shape: {self.windows[0][0].shape}\")\n            print(f\"Rotation features shape: {self.windows[0][1].shape}\")\n\n    def __getitem__(self, idx):\n        acc, rot, mask, label = self.windows[idx]\n        \n        # Add modality-specific noise augmentation for training\n        if self.mode == 'train':\n            # More noise for subtle gestures to improve robustness\n            if self.gesture_types[idx] == 'subtle':\n                acc_noise = np.random.normal(0, 0.03, acc.shape).astype(np.float32)\n                rot_noise = np.random.normal(0, 0.01, rot.shape).astype(np.float32)\n            else:\n                acc_noise = np.random.normal(0, 0.02, acc.shape).astype(np.float32)\n                rot_noise = np.random.normal(0, 0.005, rot.shape).astype(np.float32)\n            \n            acc = acc + acc_noise\n            rot = rot + rot_noise\n        \n        return {\n            \"acc\": torch.tensor(acc, dtype=torch.float32),\n            \"rot\": torch.tensor(rot, dtype=torch.float32),\n            \"mask\": torch.tensor(mask, dtype=torch.bool),\n            \"label\": torch.tensor(label, dtype=torch.long)\n        }\n\n# ============================================================\n# SIMPLIFIED BUT EFFECTIVE MODEL\n# ============================================================\n\nclass ModalityEncoder(nn.Module):\n    \"\"\"Encoder for each modality with attention\"\"\"\n    def __init__(self, input_dim, d_model=128, nhead=8, dropout=0.3, num_layers=2):\n        super().__init__()\n        \n        self.d_model = d_model\n        self.input_proj = nn.Linear(input_dim, d_model)\n        \n        # Positional encoding\n        self.pos_emb = nn.Parameter(torch.randn(1, SEQ_LEN, d_model) * 0.02)\n        \n        # Transformer encoder\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=d_model * 4,\n            dropout=dropout,\n            batch_first=True,\n            activation='gelu'\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers)\n        \n        # Attention pooling\n        self.attention_pool = nn.Sequential(\n            nn.Linear(d_model, d_model // 2),\n            nn.Tanh(),\n            nn.Linear(d_model // 2, 1)\n        )\n        \n        self.dropout = nn.Dropout(dropout)\n        self.norm = nn.LayerNorm(d_model)\n\n    def forward(self, x, mask):\n        # Input projection\n        x = self.input_proj(x)\n        x = x + self.pos_emb\n        x = self.dropout(x)\n        x = self.norm(x)\n        \n        # Transformer encoder\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        \n        # Attention pooling\n        attn_weights = self.attention_pool(x)\n        attn_weights = attn_weights.masked_fill(~mask.unsqueeze(-1), -1e9)\n        attn_weights = F.softmax(attn_weights, dim=1)\n        \n        # Weighted sum\n        pooled = torch.sum(attn_weights * x, dim=1)\n        \n        return pooled\n\nclass ImprovedFusedTransformer(nn.Module):\n    \"\"\"Simplified but effective fusion model\"\"\"\n    def __init__(self, num_classes, acc_input_dim=12, rot_input_dim=12, \n                 d_model=128, dropout=0.3):\n        super().__init__()\n        \n        # Modality encoders\n        self.acc_encoder = ModalityEncoder(acc_input_dim, d_model, dropout=dropout)\n        self.rot_encoder = ModalityEncoder(rot_input_dim, d_model, dropout=dropout)\n        \n        # Gated fusion\n        self.gate_network = nn.Sequential(\n            nn.Linear(d_model * 2, d_model),\n            nn.GELU(),\n            nn.Linear(d_model, 2),\n            nn.Softmax(dim=-1)\n        )\n        \n        # Classification head\n        self.classifier = nn.Sequential(\n            nn.LayerNorm(d_model * 2),\n            nn.Linear(d_model * 2, d_model),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, num_classes)\n        )\n        \n        # Initialize weights\n        self._init_weights()\n    \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.LayerNorm):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n    \n    def forward(self, acc, rot, mask):\n        # Encode each modality\n        acc_features = self.acc_encoder(acc, mask)\n        rot_features = self.rot_encoder(rot, mask)\n        \n        # Learn modality importance\n        combined = torch.cat([acc_features, rot_features], dim=1)\n        gate_weights = self.gate_network(combined)\n        \n        # Gated combination\n        gated_acc = gate_weights[:, 0:1] * acc_features\n        gated_rot = gate_weights[:, 1:2] * rot_features\n        \n        # Combine\n        fused_features = torch.cat([gated_acc, gated_rot], dim=1)\n        \n        # Classification\n        logits = self.classifier(fused_features)\n        \n        return logits, gate_weights\n\n# ============================================================\n# LOAD DATA WITH CLASS BALANCING\n# ============================================================\n\ndataset = EnhancedMultimodalDataset(CSV_PATH, window_before=10, window_after=10, mode='train')\ndataset.build_windows()  # Build windows after initialization\n\nnum_classes = len(dataset.gesture_classes)\nprint(f\"\\nNumber of classes: {num_classes}\")\n\nif dataset.windows:\n    acc_input_dim = dataset.windows[0][0].shape[1]\n    rot_input_dim = dataset.windows[0][1].shape[1]\n    print(f\"Accelerometer input dim: {acc_input_dim}\")\n    print(f\"Rotation input dim: {rot_input_dim}\")\nelse:\n    print(\"No windows created. Check dataset.\")\n    exit()\n\n# Compute class weights for imbalance\nclass_counts = defaultdict(int)\nfor _, _, _, label in dataset.windows:\n    class_counts[label] += 1\n\nprint(f\"\\nClass distribution:\")\nfor class_id, count in sorted(class_counts.items()):\n    print(f\"  Class {class_id} ({dataset.gesture_classes[class_id]}): {count} samples\")\n\nclass_weights = torch.zeros(num_classes)\nfor class_id, count in class_counts.items():\n    class_weights[class_id] = 1.0 / count\nclass_weights = class_weights / class_weights.sum()\n\n# Split data\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_ds, test_ds = random_split(dataset, [train_size, test_size])\n\n# Create weighted sampler for training\ntrain_labels = [dataset.windows[i][3] for i in train_ds.indices]\nsample_weights = [class_weights[label] for label in train_labels]\nsampler = WeightedRandomSampler(sample_weights, len(train_labels))\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, sampler=sampler)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE)\n\n# ============================================================\n# TRAINING WITH FOCUS ON SUBTLE GESTURES\n# ============================================================\n\nmodel = ImprovedFusedTransformer(\n    num_classes=num_classes,\n    acc_input_dim=acc_input_dim,\n    rot_input_dim=rot_input_dim,\n    d_model=D_MODEL,\n    dropout=DROPOUT\n).to(DEVICE)\n\n# Count parameters\ntotal_params = sum(p.numel() for p in model.parameters())\nprint(f\"Total parameters: {total_params:,}\")\n\n# Focal loss for handling class imbalance and hard examples\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=None, gamma=2.0):\n        super().__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        \n    def forward(self, logits, targets):\n        ce_loss = F.cross_entropy(logits, targets, reduction='none')\n        pt = torch.exp(-ce_loss)\n        focal_loss = ((1 - pt) ** self.gamma) * ce_loss\n        \n        if self.alpha is not None:\n            alpha_t = self.alpha[targets]\n            focal_loss = alpha_t * focal_loss\n            \n        return focal_loss.mean()\n\n# Use focal loss with higher weight for subtle gestures\nsubtle_indices = [i for i, gtype in dataset.gesture_type.items() if gtype == 'subtle']\nalpha = torch.ones(num_classes)\nfor idx in subtle_indices:\n    alpha[idx] = 2.0  # Higher weight for subtle gestures\n\ncriterion = FocalLoss(alpha=alpha.to(DEVICE), gamma=2.0)\n\noptimizer = torch.optim.AdamW(\n    model.parameters(),\n    lr=LR,\n    weight_decay=1e-4,\n    betas=(0.9, 0.999)\n)\n\n# Learning rate scheduler\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n    optimizer, \n    T_max=EPOCHS * len(train_loader),\n    eta_min=LR * 0.01\n)\n\n# Training metrics\ntrain_losses = []\nval_losses = []\nval_accs = []\nbest_val_acc = 0\npatience = 10\npatience_counter = 0\n\nprint(f\"\\nStarting training for {EPOCHS} epochs...\")\n\nfor epoch in range(EPOCHS):\n    # Training\n    model.train()\n    epoch_loss = 0\n    correct = 0\n    total = 0\n    \n    for batch_idx, batch in enumerate(train_loader):\n        acc = batch[\"acc\"].to(DEVICE)\n        rot = batch[\"rot\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        label = batch[\"label\"].to(DEVICE)\n        \n        optimizer.zero_grad()\n        logits, _ = model(acc, rot, mask)\n        loss = criterion(logits, label)\n        \n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        optimizer.step()\n        scheduler.step()\n        \n        epoch_loss += loss.item()\n        preds = logits.argmax(dim=1)\n        correct += (preds == label).sum().item()\n        total += label.size(0)\n        \n        # Print progress every 10 batches\n        if batch_idx % 10 == 0:\n            print(f\"Epoch {epoch+1} | Batch {batch_idx} | Loss: {loss.item():.4f}\")\n    \n    train_loss = epoch_loss / len(train_loader)\n    train_acc = correct / total\n    train_losses.append(train_loss)\n    \n    # Validation\n    model.eval()\n    val_loss = 0\n    val_correct = 0\n    val_total = 0\n    subtle_correct = 0\n    subtle_total = 0\n    \n    with torch.no_grad():\n        for batch in test_loader:\n            acc = batch[\"acc\"].to(DEVICE)\n            rot = batch[\"rot\"].to(DEVICE)\n            mask = batch[\"mask\"].to(DEVICE)\n            label = batch[\"label\"].to(DEVICE)\n            \n            logits, _ = model(acc, rot, mask)\n            loss = criterion(logits, label)\n            \n            val_loss += loss.item()\n            preds = logits.argmax(dim=1)\n            val_correct += (preds == label).sum().item()\n            val_total += label.size(0)\n            \n            # Track subtle gesture accuracy\n            for i in range(len(label)):\n                if label[i].item() in subtle_indices:\n                    subtle_total += 1\n                    if preds[i] == label[i]:\n                        subtle_correct += 1\n    \n    val_loss = val_loss / len(test_loader)\n    val_acc = val_correct / val_total\n    subtle_acc = subtle_correct / max(subtle_total, 1)\n    \n    val_losses.append(val_loss)\n    val_accs.append(val_acc)\n    \n    print(f\"\\nEpoch {epoch+1}/{EPOCHS}\")\n    print(f\"  Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.4f}\")\n    print(f\"  Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.4f}\")\n    if subtle_total > 0:\n        print(f\"  Subtle Gesture Acc: {subtle_acc:.4f} ({subtle_correct}/{subtle_total})\")\n    \n    # Early stopping with patience\n    if val_acc > best_val_acc:\n        best_val_acc = val_acc\n        patience_counter = 0\n        torch.save(model.state_dict(), 'best_model.pth')\n        print(f\"  ✓ Saved best model (Val Acc: {val_acc:.4f})\")\n    else:\n        patience_counter += 1\n        if patience_counter >= patience:\n            print(f\"  ⚠ Early stopping at epoch {epoch+1}\")\n            break\n\n# ============================================================\n# EVALUATION AND ANALYSIS\n# ============================================================\n\nmodel.load_state_dict(torch.load('best_model.pth'))\nmodel.eval()\n\nall_preds, all_labels, all_probs = [], [], []\ngesture_results = defaultdict(lambda: {'correct': 0, 'total': 0})\n\nwith torch.no_grad():\n    for batch in test_loader:\n        acc = batch[\"acc\"].to(DEVICE)\n        rot = batch[\"rot\"].to(DEVICE)\n        mask = batch[\"mask\"].to(DEVICE)\n        labels = batch[\"label\"].numpy()\n        \n        logits, _ = model(acc, rot, mask)\n        probs = F.softmax(logits, dim=1)\n        preds = logits.argmax(dim=1).cpu().numpy()\n        \n        all_preds.extend(preds)\n        all_labels.extend(labels)\n        all_probs.extend(probs.cpu().numpy())\n        \n        for true_label, pred_label in zip(labels, preds):\n            gesture = dataset.gesture_classes[true_label]\n            gesture_results[gesture]['total'] += 1\n            if true_label == pred_label:\n                gesture_results[gesture]['correct'] += 1\n\nall_preds = np.array(all_preds)\nall_labels = np.array(all_labels)\n\n# ============================================================\n# IMPROVED METRICS WITH DETAILED ANALYSIS\n# ============================================================\n\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n\noverall_acc = accuracy_score(all_labels, all_preds)\nprint(f\"\\n{'='*70}\")\nprint(f\"IMPROVED RESULTS\")\nprint(f\"{'='*70}\")\nprint(f\"\\nOverall accuracy: {overall_acc:.4f}\")\n\n# Per-gesture accuracy with type classification\nprint(\"\\nGesture Accuracy by Type:\")\nprint(\"-\" * 80)\nprint(f\"{'Gesture':30s} | {'Type':10s} | {'Acc':6s} | {'Samples':8s}\")\nprint(\"-\" * 80)\n\nfor gesture in dataset.gesture_classes:\n    if gesture in gesture_results:\n        acc = gesture_results[gesture]['correct'] / max(gesture_results[gesture]['total'], 1)\n        gtype = dataset.gesture_type.get(dataset.le.transform([gesture])[0], 'other')\n        print(f\"{gesture:30s} | {gtype:10s} | {acc:.4f} | {gesture_results[gesture]['total']:8d}\")\n\n# Calculate accuracy by gesture type\ntype_accuracies = defaultdict(lambda: {'correct': 0, 'total': 0})\nfor gesture, results in gesture_results.items():\n    gtype = dataset.gesture_type.get(dataset.le.transform([gesture])[0], 'other')\n    type_accuracies[gtype]['correct'] += results['correct']\n    type_accuracies[gtype]['total'] += results['total']\n\nprint(\"\\nAccuracy by Gesture Type:\")\nprint(\"-\" * 40)\nfor gtype in ['subtle', 'large', 'other']:\n    if type_accuracies[gtype]['total'] > 0:\n        acc = type_accuracies[gtype]['correct'] / type_accuracies[gtype]['total']\n        print(f\"{gtype:10s}: {acc:.4f} ({type_accuracies[gtype]['correct']}/{type_accuracies[gtype]['total']})\")\n\n# Confusion matrix focusing on confused gestures\nconf = confusion_matrix(all_labels, all_preds)\n\n# Identify most confused pairs\nconf_pairs = []\nfor i in range(num_classes):\n    for j in range(num_classes):\n        if i != j and conf[i, j] > 0:\n            conf_pairs.append((conf[i, j], i, j))\n\nconf_pairs.sort(reverse=True)\n\nprint(\"\\nMost confused gesture pairs:\")\nprint(\"-\" * 60)\nfor count, i, j in conf_pairs[:10]:\n    print(f\"{dataset.gesture_classes[i]:30s} → {dataset.gesture_classes[j]:30s}: {count}\")\n\n# ============================================================\n# VISUALIZE IMPROVEMENTS\n# ============================================================\n\n# Plot training curves\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n\nax1.plot(train_losses, label='Training Loss')\nax1.plot(val_losses, label='Validation Loss')\nax1.set_xlabel('Epoch')\nax1.set_ylabel('Loss')\nax1.set_title('Training and Validation Loss')\nax1.legend()\nax1.grid(True, alpha=0.3)\n\nax2.plot(val_accs, label='Validation Accuracy', color='green')\nax2.axhline(y=best_val_acc, color='r', linestyle='--', label=f'Best: {best_val_acc:.4f}')\nax2.set_xlabel('Epoch')\nax2.set_ylabel('Accuracy')\nax2.set_title('Validation Accuracy')\nax2.legend()\nax2.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()\n\n# Bar chart of gesture accuracies\ngestures = []\naccuracies = []\ncolors = []\n\nfor gesture in dataset.gesture_classes:\n    if gesture in gesture_results and gesture_results[gesture]['total'] > 0:\n        gestures.append(gesture)\n        acc = gesture_results[gesture]['correct'] / gesture_results[gesture]['total']\n        accuracies.append(acc)\n        gtype = dataset.gesture_type.get(dataset.le.transform([gesture])[0], 'other')\n        if gtype == 'subtle':\n            colors.append('red')\n        elif gtype == 'large':\n            colors.append('green')\n        else:\n            colors.append('blue')\n\n# Sort by accuracy\nsorted_indices = np.argsort(accuracies)\ngestures = [gestures[i] for i in sorted_indices]\naccuracies = [accuracies[i] for i in sorted_indices]\ncolors = [colors[i] for i in sorted_indices]\n\nplt.figure(figsize=(12, 8))\nbars = plt.barh(range(len(gestures)), accuracies, color=colors)\nplt.xlabel('Accuracy')\nplt.ylabel('Gesture')\nplt.title('Gesture Recognition Accuracy (Red=Subtle, Green=Large)')\nplt.yticks(range(len(gestures)), gestures)\nplt.xlim([0, 1.0])\nplt.grid(True, alpha=0.3, axis='x')\n\n# Add accuracy values on bars\nfor i, (bar, acc) in enumerate(zip(bars, accuracies)):\n    plt.text(bar.get_width() + 0.01, bar.get_y() + bar.get_height()/2, \n             f'{acc:.3f}', va='center')\n\nplt.tight_layout()\nplt.show()\n\n# ============================================================\n# TOP-K ACCURACY\n# ============================================================\n\ncorrect_topk = 0\nfor i in range(len(all_labels)):\n    label = all_labels[i]\n    prob = all_probs[i]\n    top_k_indices = np.argsort(prob)[-TOP_K:][::-1]\n    if label in top_k_indices:\n        correct_topk += 1\n\ntopk_acc = correct_topk / len(all_labels)\nprint(f\"\\nTop-{TOP_K} accuracy: {topk_acc:.4f}\")\n\nprint(f\"\\n{'='*70}\")\nprint(\"ANALYSIS COMPLETE\")\nprint(f\"{'='*70}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:42:40.531602Z","iopub.execute_input":"2026-02-02T17:42:40.53182Z","iopub.status.idle":"2026-02-02T17:44:44.131674Z","shell.execute_reply.started":"2026-02-02T17:42:40.531803Z","shell.execute_reply":"2026-02-02T17:44:44.131057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# TEMPORAL CNN + TRANSFORMER GESTURE RECOGNITION MODEL\n# Combines local pattern extraction (CNN) with global context (Transformer)\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, random_split, WeightedRandomSampler\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import defaultdict\nfrom scipy.spatial.transform import Rotation as R\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nBATCH_SIZE = 64\nSEQ_LEN = 41\nEPOCHS = 30\nLR = 2e-4\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint(\"Using device:\", DEVICE)\ntorch.manual_seed(42)\nnp.random.seed(42)\n\n# ============================================================\n# DATA HELPERS\n# ============================================================\n\ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    \"\"\"Intelligently handle missing quaternion values\"\"\"\n    rot_cleaned = rot_data.copy()\n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        if missing_count == 0:\n            norm = np.linalg.norm(row)\n            rot_cleaned[i] = row / norm if norm > 1e-8 else [1.0, 0.0, 0.0, 0.0]\n        elif missing_count == 1:\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            sum_squares = np.sum(valid_values**2)\n            missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n            if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n                if rot_cleaned[i-1, missing_idx] < 0:\n                    missing_value = -missing_value\n            rot_cleaned[i, missing_idx] = missing_value\n            rot_cleaned[i, ~np.isnan(row)] = valid_values\n        else:\n            rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n    return rot_cleaned\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    \"\"\"Convert acceleration from device to world coordinates\"\"\"\n    try:\n        rot_scipy = rot[:, [1, 2, 3, 0]]  # [w,x,y,z] -> [x,y,z,w] for scipy\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        if np.any(norms < 1e-8):\n            mask = norms < 1e-8\n            rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0]\n        r = R.from_quat(rot_scipy)\n        acc_world = r.apply(acc)\n        return acc_world\n    except Exception:\n        return acc.copy()\n\ndef add_engineered_features(df):\n    \"\"\"Add engineered features to the dataframe\"\"\"\n    # Acceleration magnitude and jerk\n    df['acc_mag'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n    df['jerk_x'] = df.groupby('sequence_id')['acc_x'].diff().fillna(0) * 200\n    df['jerk_y'] = df.groupby('sequence_id')['acc_y'].diff().fillna(0) * 200\n    df['jerk_z'] = df.groupby('sequence_id')['acc_z'].diff().fillna(0) * 200\n    df['jerk_mag'] = np.sqrt(df['jerk_x']**2 + df['jerk_y']**2 + df['jerk_z']**2)\n    \n    # Angular velocity approximation\n    df['rot_diff_w'] = df.groupby('sequence_id')['rot_w'].diff().fillna(0) * 200\n    df['rot_diff_x'] = df.groupby('sequence_id')['rot_x'].diff().fillna(0) * 200\n    df['rot_diff_y'] = df.groupby('sequence_id')['rot_y'].diff().fillna(0) * 200\n    df['rot_diff_z'] = df.groupby('sequence_id')['rot_z'].diff().fillna(0) * 200\n    df['rot_diff_mag'] = np.sqrt(df['rot_diff_w']**2 + df['rot_diff_x']**2 + \n                                 df['rot_diff_y']**2 + df['rot_diff_z']**2)\n    \n    # Normalize quaternions\n    df['rot_mag'] = np.sqrt(df['rot_w']**2 + df['rot_x']**2 + df['rot_y']**2 + df['rot_z']**2)\n    mask = df['rot_mag'] < 1e-6\n    df.loc[~mask, ['rot_w', 'rot_x', 'rot_y', 'rot_z']] = df.loc[~mask, ['rot_w', 'rot_x', 'rot_y', 'rot_z']].div(\n        df.loc[~mask, 'rot_mag'], axis=0\n    )\n    df.loc[mask, ['rot_w', 'rot_x', 'rot_y', 'rot_z']] = [1.0, 0.0, 0.0, 0.0]\n    \n    # Rolling statistics for temporal patterns\n    df['acc_mag_rolling_mean'] = df.groupby('sequence_id')['acc_mag'].transform(\n        lambda x: x.rolling(window=5, min_periods=1).mean()\n    )\n    df['acc_mag_rolling_std'] = df.groupby('sequence_id')['acc_mag'].transform(\n        lambda x: x.rolling(window=5, min_periods=1).std()\n    )\n    \n    return df\n\n# ============================================================\n# ENHANCED DATASET\n# ============================================================\n\nclass GestureDataset(Dataset):\n    def __init__(self, df, feature_cols, seq_len=21, mode='train'):\n        self.seq_len = seq_len\n        self.feature_cols = feature_cols\n        self.mode = mode\n        \n        # Remove bad subjects\n        df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n        \n        # Add engineered features\n        df = add_engineered_features(df)\n        \n        # Encode gestures\n        self.gesture_encoder = LabelEncoder()\n        df['gesture_encoded'] = self.gesture_encoder.fit_transform(df['gesture'])\n        self.gesture_classes = self.gesture_encoder.classes_.tolist()\n        \n        print(f\"Gesture classes: {len(self.gesture_classes)}\")\n        \n        # Group by sequence\n        self.sequences = list(df.groupby('sequence_id'))\n        self.windows = []\n        \n        for seq_id, seq in self.sequences:\n            seq = seq.reset_index(drop=True)\n            \n            # Find gesture phase\n            gesture_rows = seq[seq['phase'] == 'Gesture']\n            if gesture_rows.empty:\n                # Use middle of sequence\n                center = len(seq) // 2\n            else:\n                center = gesture_rows.index[0]\n            \n            # Create window around gesture\n            start = max(0, center - (seq_len // 2))\n            end = min(len(seq), center + (seq_len // 2) + 1)\n            \n            # Extract features\n            features = seq.iloc[start:end][feature_cols].values.astype(np.float32)\n            \n            # Handle missing values\n            features = np.nan_to_num(features, nan=0.0)\n            \n            # Compute world acceleration\n            if 'acc_x' in feature_cols and 'rot_w' in feature_cols:\n                acc_idx = [feature_cols.index(c) for c in ['acc_x', 'acc_y', 'acc_z']]\n                rot_idx = [feature_cols.index(c) for c in ['rot_w', 'rot_x', 'rot_y', 'rot_z']]\n                \n                # Handle missing quaternions\n                rot_features = features[:, rot_idx].copy()\n                rot_features = handle_quaternion_missing_values(rot_features)\n                \n                # Compute world acceleration\n                acc_world = compute_world_acceleration(\n                    features[:, acc_idx],\n                    rot_features\n                )\n                features[:, acc_idx] = acc_world\n                features[:, rot_idx] = rot_features\n            \n            # Pad if necessary\n            if len(features) < seq_len:\n                pad = np.zeros((seq_len - len(features), features.shape[1]), dtype=np.float32)\n                features = np.vstack([pad, features])\n                pad_mask = np.zeros(seq_len, dtype=np.bool_)\n                pad_mask[-len(features):] = True\n            elif len(features) > seq_len:\n                features = features[-seq_len:]\n                pad_mask = np.ones(seq_len, dtype=np.bool_)\n            else:\n                pad_mask = np.ones(seq_len, dtype=np.bool_)\n            \n            # Get gesture label\n            gesture_label = seq['gesture_encoded'].iloc[0]\n            \n            self.windows.append({\n                'features': features,\n                'pad_mask': pad_mask,\n                'gesture': gesture_label,\n                'sequence_id': seq_id\n            })\n        \n        print(f\"Created {len(self.windows)} windows\")\n        \n        # Compute class weights for imbalance\n        self._compute_class_weights()\n    \n    def _compute_class_weights(self):\n        \"\"\"Compute class weights for imbalanced gesture classes\"\"\"\n        gesture_counts = defaultdict(int)\n        for window in self.windows:\n            gesture_counts[window['gesture']] += 1\n        \n        total = len(self.windows)\n        self.gesture_weights = torch.zeros(len(self.gesture_classes))\n        for class_id, count in gesture_counts.items():\n            # Inverse frequency weighting with smoothing\n            self.gesture_weights[class_id] = total / (len(gesture_counts) * count)\n    \n    def __len__(self):\n        return len(self.windows)\n    \n    def __getitem__(self, idx):\n        window = self.windows[idx]\n        \n        features = window['features'].copy()\n        \n        # Data augmentation for training\n        if self.mode == 'train':\n            # Add noise\n            noise = np.random.normal(0, 0.02, features.shape).astype(np.float32)\n            features = features + noise\n            \n            # Random scaling\n            scale = np.random.uniform(0.95, 1.05)\n            features = features * scale\n            \n            # Random time warping (slight)\n            if np.random.random() > 0.5:\n                # Slightly stretch or compress the sequence\n                original_len = features.shape[0]\n                new_len = int(original_len * np.random.uniform(0.95, 1.05))\n                if new_len != original_len:\n                    # Simple linear interpolation\n                    indices = np.linspace(0, original_len-1, new_len)\n                    features = np.array([np.interp(indices, range(original_len), features[:, i]) \n                                        for i in range(features.shape[1])]).T\n        \n        return {\n            'X': torch.tensor(features, dtype=torch.float32),\n            'mask': torch.tensor(window['pad_mask'], dtype=torch.bool),\n            'gesture': torch.tensor(window['gesture'], dtype=torch.long)\n        }\n\n# ============================================================\n# TEMPORAL CNN + TRANSFORMER MODEL\n# ============================================================\n\nclass TemporalBlock(nn.Module):\n    \"\"\"Basic temporal block with dilation for multi-scale receptive fields\"\"\"\n    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, dilation=1, dropout=0.2):\n        super().__init__()\n        padding = (kernel_size - 1) * dilation // 2\n        \n        self.conv1 = nn.Conv1d(\n            in_channels, out_channels, kernel_size,\n            stride=stride, padding=padding, dilation=dilation\n        )\n        self.bn1 = nn.BatchNorm1d(out_channels)\n        \n        self.conv2 = nn.Conv1d(\n            out_channels, out_channels, kernel_size,\n            stride=1, padding=padding, dilation=dilation\n        )\n        self.bn2 = nn.BatchNorm1d(out_channels)\n        \n        self.downsample = nn.Conv1d(in_channels, out_channels, 1) if in_channels != out_channels else None\n        self.dropout = nn.Dropout(dropout)\n    \n    def forward(self, x):\n        identity = x\n        if self.downsample is not None:\n            identity = self.downsample(x)\n        \n        out = F.gelu(self.bn1(self.conv1(x)))\n        out = self.dropout(out)\n        out = self.bn2(self.conv2(out))\n        \n        out += identity\n        out = F.gelu(out)\n        out = self.dropout(out)\n        \n        return out\n\nclass MultiScaleTemporalCNN(nn.Module):\n    \"\"\"Multi-scale temporal CNN with parallel dilated convolutions\"\"\"\n    def __init__(self, in_channels, hidden_channels=64, dropout=0.2):\n        super().__init__()\n        \n        # Initial projection\n        self.init_conv = nn.Conv1d(in_channels, hidden_channels, kernel_size=1)\n        self.init_bn = nn.BatchNorm1d(hidden_channels)\n        \n        # Parallel dilated convolutions for different receptive fields\n        self.dilated_convs = nn.ModuleList([\n            nn.Sequential(\n                TemporalBlock(hidden_channels, hidden_channels, dilation=1, dropout=dropout),\n                TemporalBlock(hidden_channels, hidden_channels, dilation=2, dropout=dropout),\n            ),\n            nn.Sequential(\n                TemporalBlock(hidden_channels, hidden_channels, dilation=2, dropout=dropout),\n                TemporalBlock(hidden_channels, hidden_channels, dilation=4, dropout=dropout),\n            ),\n            nn.Sequential(\n                TemporalBlock(hidden_channels, hidden_channels, dilation=3, dropout=dropout),\n                TemporalBlock(hidden_channels, hidden_channels, dilation=6, dropout=dropout),\n            )\n        ])\n        \n        # Feature fusion\n        self.fusion = nn.Sequential(\n            nn.Conv1d(hidden_channels * 3, hidden_channels * 2, kernel_size=1),\n            nn.BatchNorm1d(hidden_channels * 2),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Conv1d(hidden_channels * 2, hidden_channels, kernel_size=1),\n            nn.BatchNorm1d(hidden_channels),\n            nn.GELU()\n        )\n        \n        # Temporal pooling\n        self.temporal_pool = nn.AdaptiveAvgPool1d(1)\n    \n    def forward(self, x):\n        # x: (batch, seq_len, channels)\n        x = x.transpose(1, 2)  # -> (batch, channels, seq_len)\n        \n        # Initial projection\n        x = F.gelu(self.init_bn(self.init_conv(x)))\n        \n        # Parallel dilated convolutions\n        branch_outputs = []\n        for conv_path in self.dilated_convs:\n            branch_outputs.append(conv_path(x))\n        \n        # Concatenate and fuse\n        x = torch.cat(branch_outputs, dim=1)\n        x = self.fusion(x)\n        \n        # Temporal pooling\n        x = self.temporal_pool(x)  # -> (batch, channels, 1)\n        x = x.squeeze(-1)  # -> (batch, channels)\n        \n        return x\n\nclass AdaptiveTransformerEncoder(nn.Module):\n    \"\"\"Transformer encoder with adaptive input size\"\"\"\n    def __init__(self, d_model=256, nhead=8, num_layers=3, dropout=0.1):\n        super().__init__()\n        \n        self.d_model = d_model\n        \n        # Transformer layers\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=d_model * 4,\n            dropout=dropout,\n            batch_first=True,\n            activation='gelu'\n        )\n        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        \n        # Adaptive pooling\n        self.pooling = nn.Sequential(\n            nn.Linear(d_model, d_model // 2),\n            nn.Tanh(),\n            nn.Linear(d_model // 2, 1)\n        )\n        \n        self.dropout = nn.Dropout(dropout)\n    \n    def forward(self, x, mask=None):\n        # x: (batch, seq_len, d_model)\n        batch_size = x.size(0)\n        \n        # Add positional encoding (learned)\n        seq_len = x.size(1)\n        if not hasattr(self, 'pos_emb') or self.pos_emb.size(1) != seq_len:\n            self.pos_emb = nn.Parameter(torch.randn(1, seq_len, self.d_model) * 0.02).to(x.device)\n        \n        x = x + self.pos_emb\n        x = self.dropout(x)\n        \n        # Transformer\n        if mask is not None:\n            x = self.transformer(x, src_key_padding_mask=~mask)\n        else:\n            x = self.transformer(x)\n        \n        # Adaptive pooling\n        attn_weights = self.pooling(x)  # (batch, seq_len, 1)\n        if mask is not None:\n            attn_weights = attn_weights.masked_fill(~mask.unsqueeze(-1), -1e9)\n        \n        attn_weights = F.softmax(attn_weights, dim=1)\n        pooled = torch.sum(attn_weights * x, dim=1)  # (batch, d_model)\n        \n        return pooled, attn_weights\n\nclass TemporalCNNTransformer(nn.Module):\n    \"\"\"Main model: Temporal CNN + Transformer\"\"\"\n    def __init__(self, input_dim, num_classes, cnn_hidden=64, transformer_dim=256, \n                 transformer_heads=8, transformer_layers=3, dropout=0.2):\n        super().__init__()\n        \n        # ========== MODALITY-SPECIFIC TEMPORAL CNNs ==========\n        # Accelerometer CNN\n        self.acc_cnn = MultiScaleTemporalCNN(\n            in_channels=3,  # x, y, z\n            hidden_channels=cnn_hidden // 2,\n            dropout=dropout\n        )\n        \n        # Rotation CNN\n        self.rot_cnn = MultiScaleTemporalCNN(\n            in_channels=4,  # w, x, y, z\n            hidden_channels=cnn_hidden // 2,\n            dropout=dropout\n        )\n        \n        # Derived features CNN\n        self.derived_cnn = MultiScaleTemporalCNN(\n            in_channels=input_dim - 7,  # All other features\n            hidden_channels=cnn_hidden // 2,\n            dropout=dropout\n        )\n        \n        # ========== FEATURE FUSION ==========\n        self.feature_fusion = nn.Sequential(\n            nn.Linear(cnn_hidden * 3 // 2, transformer_dim),\n            nn.LayerNorm(transformer_dim),\n            nn.GELU(),\n            nn.Dropout(dropout)\n        )\n        \n        # ========== TRANSFORMER ENCODER ==========\n        self.transformer_encoder = AdaptiveTransformerEncoder(\n            d_model=transformer_dim,\n            nhead=transformer_heads,\n            num_layers=transformer_layers,\n            dropout=dropout\n        )\n        \n        # ========== CLASSIFICATION HEAD ==========\n        self.classifier = nn.Sequential(\n            nn.LayerNorm(transformer_dim),\n            nn.Linear(transformer_dim, transformer_dim // 2),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(transformer_dim // 2, transformer_dim // 4),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.5),\n            nn.Linear(transformer_dim // 4, num_classes)\n        )\n        \n        # Initialize weights\n        self._init_weights()\n    \n    def _init_weights(self):\n        for name, module in self.named_modules():\n            if isinstance(module, nn.Linear):\n                nn.init.xavier_uniform_(module.weight)\n                if module.bias is not None:\n                    nn.init.zeros_(module.bias)\n            elif isinstance(module, nn.LayerNorm):\n                nn.init.ones_(module.weight)\n                nn.init.zeros_(module.bias)\n            elif isinstance(module, nn.Conv1d):\n                nn.init.kaiming_normal_(module.weight, nonlinearity='relu')\n                if module.bias is not None:\n                    nn.init.zeros_(module.bias)\n    \n    def forward(self, x, mask=None):\n        # x: (batch, seq_len, input_dim)\n        batch_size, seq_len, input_dim = x.shape\n        \n        # Split into modalities\n        # Assuming first 3 features are acceleration\n        acc_features = x[:, :, :3]\n        # Next 4 features are rotation\n        rot_features = x[:, :, 3:7]\n        # Remaining are derived features\n        derived_features = x[:, :, 7:]\n        \n        # Process each modality with its own CNN\n        # Need to reshape for CNN: (batch, seq_len, channels) -> (batch, channels, seq_len)\n        \n        # Process acceleration\n        acc_processed = self.acc_cnn(acc_features)  # (batch, cnn_hidden//2)\n        \n        # Process rotation\n        rot_processed = self.rot_cnn(rot_features)  # (batch, cnn_hidden//2)\n        \n        # Process derived features (if any)\n        if derived_features.size(2) > 0:\n            derived_processed = self.derived_cnn(derived_features)  # (batch, cnn_hidden//2)\n        else:\n            derived_processed = torch.zeros(batch_size, self.derived_cnn.fusion[0].in_channels // 3, \n                                          device=x.device)\n        \n        # Concatenate modality features\n        fused_features = torch.cat([acc_processed, rot_processed, derived_processed], dim=1)\n        \n        # Project to transformer dimension\n        transformer_input = self.feature_fusion(fused_features)\n        transformer_input = transformer_input.unsqueeze(1)  # (batch, 1, transformer_dim)\n        \n        # Create a dummy mask for transformer (since we have sequence length 1 after CNN)\n        if mask is not None:\n            # We need to adapt the mask for the transformer input\n            # For simplicity, we'll use a mask of all True (since we have single token)\n            transformer_mask = torch.ones(batch_size, 1, dtype=torch.bool, device=x.device)\n        else:\n            transformer_mask = None\n        \n        # Transformer encoder (processes the fused representation)\n        transformer_output, attn_weights = self.transformer_encoder(\n            transformer_input, \n            mask=transformer_mask\n        )\n        \n        # Classification\n        logits = self.classifier(transformer_output)\n        \n        return logits, attn_weights\n\n# Alternative: Sequential CNN -> Transformer\nclass SequentialTemporalCNNTransformer(nn.Module):\n    \"\"\"Alternative: Sequential processing (CNN -> Transformer on reduced sequence)\"\"\"\n    def __init__(self, input_dim, num_classes, cnn_channels=64, transformer_dim=256, \n                 reduced_seq_len=10, dropout=0.2):\n        super().__init__()\n        \n        # ========== TEMPORAL CNN FOR SEQUENCE REDUCTION ==========\n        self.cnn_encoder = nn.Sequential(\n            # First conv block\n            nn.Conv1d(input_dim, cnn_channels, kernel_size=3, padding=1),\n            nn.BatchNorm1d(cnn_channels),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            \n            # Second conv block with stride for downsampling\n            nn.Conv1d(cnn_channels, cnn_channels * 2, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(cnn_channels * 2),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            \n            # Third conv block\n            nn.Conv1d(cnn_channels * 2, cnn_channels * 2, kernel_size=3, padding=1),\n            nn.BatchNorm1d(cnn_channels * 2),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            \n            # Adaptive pooling to fixed length\n            nn.AdaptiveAvgPool1d(reduced_seq_len)\n        )\n        \n        # ========== TRANSFORMER ON REDUCED SEQUENCE ==========\n        self.transformer_encoder = AdaptiveTransformerEncoder(\n            d_model=transformer_dim,\n            nhead=8,\n            num_layers=3,\n            dropout=dropout\n        )\n        \n        # Project CNN features to transformer dimension\n        self.cnn_to_transformer = nn.Sequential(\n            nn.Linear(cnn_channels * 2, transformer_dim),\n            nn.LayerNorm(transformer_dim),\n            nn.GELU(),\n            nn.Dropout(dropout)\n        )\n        \n        # ========== CLASSIFICATION HEAD ==========\n        self.classifier = nn.Sequential(\n            nn.LayerNorm(transformer_dim),\n            nn.Linear(transformer_dim, transformer_dim // 2),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.5),\n            nn.Linear(transformer_dim // 2, num_classes)\n        )\n        \n        self.reduced_seq_len = reduced_seq_len\n    \n    def forward(self, x, mask=None):\n        # x: (batch, seq_len, input_dim)\n        batch_size = x.size(0)\n        \n        # Reshape for CNN: (batch, channels, seq_len)\n        x_cnn = x.transpose(1, 2)\n        \n        # CNN encoding and downsampling\n        x_cnn = self.cnn_encoder(x_cnn)  # (batch, cnn_channels*2, reduced_seq_len)\n        x_cnn = x_cnn.transpose(1, 2)  # (batch, reduced_seq_len, cnn_channels*2)\n        \n        # Project to transformer dimension\n        x_transformer = self.cnn_to_transformer(x_cnn)\n        \n        # Adjust mask for reduced sequence length\n        if mask is not None:\n            # Downsample mask (keep only every other position due to stride)\n            reduced_mask = mask[:, ::2]  # Simple downsampling\n            # Ensure we have the right length\n            if reduced_mask.size(1) > self.reduced_seq_len:\n                reduced_mask = reduced_mask[:, :self.reduced_seq_len]\n            elif reduced_mask.size(1) < self.reduced_seq_len:\n                # Pad if needed\n                pad_len = self.reduced_seq_len - reduced_mask.size(1)\n                reduced_mask = F.pad(reduced_mask, (0, pad_len), value=False)\n        else:\n            reduced_mask = None\n        \n        # Transformer encoder\n        transformer_output, attn_weights = self.transformer_encoder(\n            x_transformer, \n            mask=reduced_mask\n        )\n        \n        # Classification\n        logits = self.classifier(transformer_output)\n        \n        return logits, attn_weights\n\n# ============================================================\n# TRAINING UTILITIES\n# ============================================================\n\nclass FocalLoss(nn.Module):\n    \"\"\"Focal loss for class imbalance\"\"\"\n    def __init__(self, alpha=None, gamma=2.0, reduction='mean'):\n        super().__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        self.reduction = reduction\n    \n    def forward(self, inputs, targets):\n        ce_loss = F.cross_entropy(inputs, targets, reduction='none')\n        pt = torch.exp(-ce_loss)\n        focal_loss = ((1 - pt) ** self.gamma) * ce_loss\n        \n        if self.alpha is not None:\n            alpha_t = self.alpha[targets]\n            focal_loss = alpha_t * focal_loss\n        \n        if self.reduction == 'mean':\n            return focal_loss.mean()\n        elif self.reduction == 'sum':\n            return focal_loss.sum()\n        else:\n            return focal_loss\n\ndef train_one_epoch(model, loader, optimizer, scheduler, device, criterion):\n    model.train()\n    total_loss = 0\n    correct = 0\n    n = 0\n    \n    for batch in loader:\n        X = batch[\"X\"].to(device)\n        mask = batch[\"mask\"].to(device)\n        gesture = batch[\"gesture\"].to(device)\n        \n        optimizer.zero_grad()\n        \n        # Forward pass\n        logits, _ = model(X, mask)\n        loss = criterion(logits, gesture)\n        \n        loss.backward()\n        \n        # Gradient clipping\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        \n        optimizer.step()\n        if scheduler is not None:\n            scheduler.step()\n        \n        # Metrics\n        total_loss += loss.item() * X.size(0)\n        preds = logits.argmax(1)\n        correct += (preds == gesture).sum().item()\n        n += X.size(0)\n    \n    return total_loss / n, correct / n\n\ndef evaluate(model, loader, device, criterion, return_predictions=False):\n    model.eval()\n    total_loss = 0\n    correct = 0\n    n = 0\n    \n    all_preds = []\n    all_labels = []\n    all_probs = []\n    \n    with torch.no_grad():\n        for batch in loader:\n            X = batch[\"X\"].to(device)\n            mask = batch[\"mask\"].to(device)\n            gesture = batch[\"gesture\"].to(device)\n            \n            # Forward pass\n            logits, _ = model(X, mask)\n            loss = criterion(logits, gesture)\n            \n            # Metrics\n            total_loss += loss.item() * X.size(0)\n            preds = logits.argmax(1)\n            correct += (preds == gesture).sum().item()\n            n += X.size(0)\n            \n            # Store for analysis\n            if return_predictions:\n                all_preds.extend(preds.cpu().numpy())\n                all_labels.extend(gesture.cpu().numpy())\n                probs = F.softmax(logits, dim=1)\n                all_probs.extend(probs.cpu().numpy())\n    \n    results = {\n        'loss': total_loss / n,\n        'accuracy': correct / n,\n    }\n    \n    if return_predictions:\n        results.update({\n            'predictions': np.array(all_preds),\n            'labels': np.array(all_labels),\n            'probabilities': np.array(all_probs)\n        })\n    \n    return results\n\n# ============================================================\n# MODEL SELECTION AND TRAINING\n# ============================================================\n\ndef main(model_type='parallel'):\n    \"\"\"Main training function with model selection\"\"\"\n    \n    # Load data\n    df = pd.read_csv(CSV_PATH)\n    \n    # Define feature columns\n    base_features = ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']\n    engineered_features = ['acc_mag', 'jerk_x', 'jerk_y', 'jerk_z', 'jerk_mag',\n                          'rot_diff_w', 'rot_diff_x', 'rot_diff_y', 'rot_diff_z', 'rot_diff_mag',\n                          'acc_mag_rolling_mean', 'acc_mag_rolling_std']\n    feature_cols = base_features + engineered_features\n    \n    # Create dataset\n    dataset = GestureDataset(df, feature_cols, seq_len=SEQ_LEN, mode='train')\n    \n    # Split dataset\n    train_size = int(0.8 * len(dataset))\n    test_size = len(dataset) - train_size\n    train_ds, test_ds = random_split(dataset, [train_size, test_size])\n    \n    # Update mode for test set\n    test_ds.dataset.mode = 'test'\n    \n    # Create weighted sampler for training\n    train_labels = [dataset.windows[i]['gesture'] for i in train_ds.indices]\n    sample_weights = [dataset.gesture_weights[label] for label in train_labels]\n    sampler = WeightedRandomSampler(sample_weights, len(train_labels))\n    \n    # Create data loaders\n    train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, sampler=sampler)\n    test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE)\n    \n    # Model parameters\n    num_classes = len(dataset.gesture_classes)\n    input_dim = len(feature_cols)\n    \n    print(f\"\\nModel Configuration:\")\n    print(f\"  Input dimension: {input_dim}\")\n    print(f\"  Number of classes: {num_classes}\")\n    print(f\"  Sequence length: {SEQ_LEN}\")\n    \n    # Select model architecture\n    if model_type == 'parallel':\n        print(\"  Architecture: Parallel Temporal CNN + Transformer\")\n        model = TemporalCNNTransformer(\n            input_dim=input_dim,\n            num_classes=num_classes,\n            cnn_hidden=64,\n            transformer_dim=256,\n            transformer_heads=8,\n            transformer_layers=3,\n            dropout=0.2\n        )\n    elif model_type == 'sequential':\n        print(\"  Architecture: Sequential CNN -> Transformer\")\n        model = SequentialTemporalCNNTransformer(\n            input_dim=input_dim,\n            num_classes=num_classes,\n            cnn_channels=64,\n            transformer_dim=256,\n            reduced_seq_len=10,\n            dropout=0.2\n        )\n    else:\n        raise ValueError(f\"Unknown model type: {model_type}\")\n    \n    model = model.to(DEVICE)\n    \n    # Count parameters\n    total_params = sum(p.numel() for p in model.parameters())\n    print(f\"  Total parameters: {total_params:,}\")\n    \n    # Loss function with class weights\n    criterion = FocalLoss(\n        alpha=dataset.gesture_weights.to(DEVICE),\n        gamma=2.0\n    )\n    \n    # Optimizer\n    optimizer = torch.optim.AdamW(\n        model.parameters(),\n        lr=LR,\n        weight_decay=1e-4,\n        betas=(0.9, 0.999)\n    )\n    \n    # Scheduler\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n        optimizer,\n        max_lr=LR,\n        epochs=EPOCHS,\n        steps_per_epoch=len(train_loader),\n        pct_start=0.1\n    )\n    \n    # Training loop\n    best_val_acc = 0\n    patience = 10\n    patience_counter = 0\n    \n    train_history = {\n        'loss': [],\n        'accuracy': [],\n        'val_loss': [],\n        'val_accuracy': []\n    }\n    \n    print(f\"\\nStarting training for {EPOCHS} epochs...\")\n    \n    for epoch in range(EPOCHS):\n        # Training\n        train_loss, train_acc = train_one_epoch(\n            model, train_loader, optimizer, scheduler, DEVICE, criterion\n        )\n        \n        # Validation\n        val_results = evaluate(\n            model, test_loader, DEVICE, criterion\n        )\n        val_loss = val_results['loss']\n        val_acc = val_results['accuracy']\n        \n        # Store history\n        train_history['loss'].append(train_loss)\n        train_history['accuracy'].append(train_acc)\n        train_history['val_loss'].append(val_loss)\n        train_history['val_accuracy'].append(val_acc)\n        \n        # Print progress\n        print(f\"\\nEpoch {epoch+1:02d}/{EPOCHS}\")\n        print(f\"  Train - Loss: {train_loss:.4f}, Acc: {train_acc:.4f}\")\n        print(f\"  Val   - Loss: {val_loss:.4f}, Acc: {val_acc:.4f}\")\n        \n        # Early stopping\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            patience_counter = 0\n            # Save model\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'val_accuracy': val_acc,\n                'train_history': train_history,\n                'gesture_classes': dataset.gesture_classes,\n                'model_type': model_type,\n                'input_dim': input_dim,\n                'num_classes': num_classes\n            }, f'best_model_{model_type}.pth')\n            print(f\"  ✓ Saved best model (Accuracy: {val_acc:.4f})\")\n        else:\n            patience_counter += 1\n            if patience_counter >= patience:\n                print(f\"  ⚠ Early stopping at epoch {epoch+1}\")\n                break\n    \n    # Load best model\n    print(f\"\\nLoading best model...\")\n    checkpoint = torch.load(f'best_model_{model_type}.pth', map_location=DEVICE, weights_only=False)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    \n    # Final evaluation\n    print(f\"\\n{'='*70}\")\n    print(f\"FINAL EVALUATION - {model_type.upper()} MODEL\")\n    print(f\"{'='*70}\")\n    \n    val_results = evaluate(\n        model, test_loader, DEVICE, criterion,\n        return_predictions=True\n    )\n    \n    print(f\"\\nResults:\")\n    print(f\"  Accuracy: {val_results['accuracy']:.4f}\")\n    print(f\"  Loss: {val_results['loss']:.4f}\")\n    \n    # Detailed analysis\n    from sklearn.metrics import classification_report, confusion_matrix\n    \n    print(f\"\\nClassification Report:\")\n    print(classification_report(\n        val_results['labels'],\n        val_results['predictions'],\n        target_names=dataset.gesture_classes,\n        digits=3\n    ))\n    \n    # Visualize results\n    import matplotlib.pyplot as plt\n    \n    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(14, 10))\n    \n    # Training curves\n    ax1.plot(train_history['loss'], label='Training Loss')\n    ax1.plot(train_history['val_loss'], label='Validation Loss')\n    ax1.set_xlabel('Epoch')\n    ax1.set_ylabel('Loss')\n    ax1.set_title('Training and Validation Loss')\n    ax1.legend()\n    ax1.grid(True, alpha=0.3)\n    \n    # Accuracy curves\n    ax2.plot(train_history['accuracy'], label='Training Accuracy')\n    ax2.plot(train_history['val_accuracy'], label='Validation Accuracy')\n    ax2.axhline(y=best_val_acc, color='r', linestyle='--', label=f'Best: {best_val_acc:.4f}')\n    ax2.set_xlabel('Epoch')\n    ax2.set_ylabel('Accuracy')\n    ax2.set_title('Training and Validation Accuracy')\n    ax2.legend()\n    ax2.grid(True, alpha=0.3)\n    \n    # Accuracy by gesture\n    gesture_accuracies = {}\n    for i, gesture in enumerate(dataset.gesture_classes):\n        mask = val_results['labels'] == i\n        if mask.sum() > 0:\n            acc = (val_results['predictions'][mask] == val_results['labels'][mask]).mean()\n            gesture_accuracies[gesture] = acc\n    \n    # Sort by accuracy\n    sorted_gestures = sorted(gesture_accuracies.items(), key=lambda x: x[1])\n    gestures = [g[0] for g in sorted_gestures]\n    accuracies = [g[1] for g in sorted_gestures]\n    \n    # Plot only top and bottom performers to avoid overcrowding\n    n_show = min(10, len(gestures))\n    show_gestures = gestures[:n_show//2] + gestures[-n_show//2:]\n    show_accuracies = accuracies[:n_show//2] + accuracies[-n_show//2:]\n    \n    colors = ['red' if i < n_show//2 else 'green' for i in range(len(show_gestures))]\n    bars = ax3.barh(range(len(show_gestures)), show_accuracies, color=colors)\n    ax3.set_xlabel('Accuracy')\n    ax3.set_ylabel('Gesture')\n    ax3.set_title('Gesture Recognition Accuracy (Red=Worst, Green=Best)')\n    ax3.set_yticks(range(len(show_gestures)))\n    ax3.set_yticklabels(show_gestures)\n    ax3.set_xlim([0, 1.0])\n    ax3.grid(True, alpha=0.3, axis='x')\n    \n    # Add accuracy values on bars\n    for i, (bar, acc) in enumerate(zip(bars, show_accuracies)):\n        ax3.text(bar.get_width() + 0.01, bar.get_y() + bar.get_height()/2,\n                f'{acc:.3f}', va='center')\n    \n    # Confusion matrix for top confused pairs\n    cm = confusion_matrix(val_results['labels'], val_results['predictions'])\n    \n    # Find most confused pairs\n    confusion_pairs = []\n    for i in range(len(dataset.gesture_classes)):\n        for j in range(len(dataset.gesture_classes)):\n            if i != j and cm[i, j] > 0:\n                confusion_pairs.append((cm[i, j], i, j))\n    \n    confusion_pairs.sort(reverse=True)\n    \n    # Show confusion matrix as text\n    ax4.axis('off')\n    ax4.set_title('Top 5 Most Confused Gesture Pairs')\n    \n    confusion_text = \"Most Confused Pairs:\\n\"\n    for count, i, j in confusion_pairs[:5]:\n        confusion_text += f\"{dataset.gesture_classes[i]:25s} -> {dataset.gesture_classes[j]:25s}: {count}\\n\"\n    \n    ax4.text(0.1, 0.5, confusion_text, fontsize=10, verticalalignment='center',\n            bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))\n    \n    plt.tight_layout()\n    plt.savefig(f'training_results_{model_type}.png', dpi=150, bbox_inches='tight')\n    plt.show()\n    \n    # Compare architectures if both are trained\n    if model_type == 'parallel':\n        print(f\"\\n{'='*70}\")\n        print(\"ARCHITECTURE COMPARISON\")\n        print(f\"{'='*70}\")\n        print(\"Parallel Temporal CNN + Transformer advantages:\")\n        print(\"1. Separate processing for different modalities\")\n        print(\"2. Multi-scale receptive fields with dilated convolutions\")\n        print(\"3. Better for capturing local patterns in each sensor modality\")\n        print(\"4. More parameters, potentially better accuracy\")\n        print(f\"\\nFinal Accuracy: {val_results['accuracy']:.4f}\")\n    \n    return model, dataset, val_results\n\n# ============================================================\n# RUN EXPERIMENTS\n# ============================================================\n\nif __name__ == \"__main__\":\n    # Choose architecture: 'parallel' or 'sequential'\n    # Parallel: Separate CNNs for each modality, then transformer\n    # Sequential: Single CNN for downsampling, then transformer\n    \n    # Run parallel architecture (recommended)\n    print(\"Training Parallel Temporal CNN + Transformer model...\")\n    model_parallel, dataset_parallel, results_parallel = main(model_type='parallel')\n    \n    # Optionally run sequential architecture for comparison\n    # print(\"\\n\" + \"=\"*70)\n    # print(\"Training Sequential CNN -> Transformer model...\")\n    # print(\"=\"*70)\n    # model_sequential, dataset_sequential, results_sequential = main(model_type='sequential')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:44:44.132693Z","iopub.execute_input":"2026-02-02T17:44:44.132918Z","iopub.status.idle":"2026-02-02T17:47:24.850133Z","shell.execute_reply.started":"2026-02-02T17:44:44.1329Z","shell.execute_reply":"2026-02-02T17:47:24.849495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# INCEPTIONTIME BASELINE FOR IMU GESTURE CLASSIFICATION\n# Kaggle-ready single file\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nBATCH_SIZE = 64\nWINDOW_BEFORE = 30\nWINDOW_AFTER = 30\nWINDOW_LEN = WINDOW_BEFORE + WINDOW_AFTER + 1\nEPOCHS = 25\nLR = 1e-3\nTOP_K = 3\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device:\", DEVICE)\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass IMUWindowDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n\n    def __init__(self, csv_path):\n        df = pd.read_csv(csv_path)\n\n        df[\"gesture\"] = df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        df[\"gesture_id\"] = self.le.fit_transform(df[\"gesture\"])\n        self.classes = self.le.classes_\n\n        self.windows = []\n\n        for _, seq in df.groupby(\"sequence_id\"):\n            seq = seq.reset_index(drop=True)\n\n            g_rows = seq[seq[\"phase\"] == \"Gesture\"]\n            if g_rows.empty:\n                continue\n\n            center = g_rows.index[0]\n            start = center - WINDOW_BEFORE\n            end = center + WINDOW_AFTER + 1\n\n            s = max(0, start)\n            e = min(len(seq), end)\n\n            acc = seq.iloc[s:e][self.ACC_COLS].values.astype(np.float32)\n            rot = seq.iloc[s:e][self.ROT_COLS].values.astype(np.float32)\n\n            if len(acc) == 0:\n                continue\n\n            # ---- normalize acc (per window)\n            acc = (acc - acc.mean(0)) / (acc.std(0) + 1e-6)\n\n            # ---- normalize quaternion\n            rot = np.nan_to_num(rot)\n            norm = np.linalg.norm(rot, axis=1, keepdims=True)\n            norm[norm < 1e-6] = 1.0\n            rot = rot / norm\n\n            x = np.concatenate([acc, rot], axis=1)  # (T, 7)\n\n            pad = np.zeros((WINDOW_LEN, 7), dtype=np.float32)\n            mask_start = s - start\n            pad[mask_start:mask_start+len(x)] = x\n\n            label = seq[\"gesture_id\"].iloc[0]\n            self.windows.append((pad, label))\n\n        print(f\"Created {len(self.windows)} windows\")\n\n    def __len__(self):\n        return len(self.windows)\n\n    def __getitem__(self, idx):\n        x, y = self.windows[idx]\n        # (C, T) for CNN\n        return (\n            torch.tensor(x.T, dtype=torch.float32),\n            torch.tensor(y, dtype=torch.long),\n        )\n\n# ============================================================\n# INCEPTION MODULE\n# ============================================================\n\nclass InceptionBlock(nn.Module):\n    def __init__(self, in_ch, out_ch):\n        super().__init__()\n\n        self.branch1 = nn.Conv1d(in_ch, out_ch, 1)\n\n        self.branch3 = nn.Sequential(\n            nn.Conv1d(in_ch, out_ch, 1),\n            nn.Conv1d(out_ch, out_ch, 3, padding=1),\n        )\n\n        self.branch5 = nn.Sequential(\n            nn.Conv1d(in_ch, out_ch, 1),\n            nn.Conv1d(out_ch, out_ch, 5, padding=2),\n        )\n\n        self.pool = nn.Sequential(\n            nn.MaxPool1d(3, stride=1, padding=1),\n            nn.Conv1d(in_ch, out_ch, 1),\n        )\n\n        self.bn = nn.BatchNorm1d(out_ch * 4)\n        self.relu = nn.ReLU()\n\n    def forward(self, x):\n        x = torch.cat([\n            self.branch1(x),\n            self.branch3(x),\n            self.branch5(x),\n            self.pool(x)\n        ], dim=1)\n        return self.relu(self.bn(x))\n\n# ============================================================\n# INCEPTIONTIME MODEL\n# ============================================================\n\nclass InceptionTime(nn.Module):\n    def __init__(self, in_channels, num_classes):\n        super().__init__()\n\n        self.block1 = InceptionBlock(in_channels, 32)\n        self.block2 = InceptionBlock(32 * 4, 32)\n        self.block3 = InceptionBlock(32 * 4, 32)\n\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        self.fc = nn.Linear(32 * 4, num_classes)\n\n    def forward(self, x):\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.pool(x).squeeze(-1)\n        return self.fc(x)\n\n# ============================================================\n# LOAD DATA\n# ============================================================\n\ndataset = IMUWindowDataset(CSV_PATH)\nnum_classes = len(dataset.classes)\n\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_ds, test_ds = random_split(dataset, [train_size, test_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE)\n\n# ============================================================\n# TRAIN\n# ============================================================\n\nmodel = InceptionTime(7, num_classes).to(DEVICE)\noptimizer = torch.optim.Adam(model.parameters(), lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total, correct, loss_sum = 0, 0, 0\n\n    for x, y in train_loader:\n        x, y = x.to(DEVICE), y.to(DEVICE)\n\n        optimizer.zero_grad()\n        logits = model(x)\n        loss = criterion(logits, y)\n        loss.backward()\n        optimizer.step()\n\n        loss_sum += loss.item()\n        correct += (logits.argmax(1) == y).sum().item()\n        total += y.size(0)\n\n    print(f\"Epoch {epoch+1:02d} | Train Acc {correct/total:.3f} | Loss {loss_sum/len(train_loader):.3f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\nmodel.eval()\nall_preds, all_labels = [], []\n\nwith torch.no_grad():\n    for x, y in test_loader:\n        x = x.to(DEVICE)\n        logits = model(x)\n        preds = logits.argmax(1).cpu().numpy()\n\n        all_preds.extend(preds)\n        all_labels.extend(y.numpy())\n\nall_preds = np.array(all_preds)\nall_labels = np.array(all_labels)\n\nacc = (all_preds == all_labels).mean()\nprint(f\"\\nOverall Accuracy: {acc:.3f}\")\n\n# ============================================================\n# CONFUSION MATRIX\n# ============================================================\n\nconf = np.zeros((num_classes, num_classes), dtype=int)\nfor t, p in zip(all_labels, all_preds):\n    conf[t, p] += 1\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(conf, xticklabels=dataset.classes, yticklabels=dataset.classes,\n            cmap=\"Blues\", fmt=\"d\")\nplt.title(\"Confusion Matrix – InceptionTime\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.tight_layout()\nplt.show()\n\n# ============================================================\n# TOP-K ACCURACY\n# ============================================================\n\ncorrect_topk = 0\nwith torch.no_grad():\n    for x, y in test_loader:\n        x, y = x.to(DEVICE), y.to(DEVICE)\n        logits = model(x)\n        topk = logits.topk(TOP_K, dim=1).indices\n        correct_topk += sum(y[i] in topk[i] for i in range(y.size(0)))\n\nprint(f\"Top-{TOP_K} Accuracy: {correct_topk / len(all_labels):.3f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:47:24.851604Z","iopub.execute_input":"2026-02-02T17:47:24.851809Z","iopub.status.idle":"2026-02-02T17:49:34.930799Z","shell.execute_reply.started":"2026-02-02T17:47:24.851794Z","shell.execute_reply":"2026-02-02T17:49:34.929993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# DeepConvLSTM IMU Baseline — FULL Kaggle Script\n# =====================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# ------------------------\n# Config\n# ------------------------\nDATA_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"  # <-- CHANGE\nWINDOW_LEN = 30\nCENTER_OFFSET = 15\nBATCH_SIZE = 64\nEPOCHS = 25\nLR = 1e-3\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint(\"Using device:\", DEVICE)\n\n# ------------------------\n# Load data\n# ------------------------\ndf = pd.read_csv(DATA_PATH)\n\n# Basic cleanup\ndf = df.sort_values([\"sequence_id\", \"row_id\"] if \"row_id\" in df.columns else [\"sequence_id\"])\ndf = df.reset_index(drop=True)\n\n# Encode labels\nle = LabelEncoder()\ndf[\"gesture_int\"] = le.fit_transform(df[\"gesture\"])\nNUM_CLASSES = len(le.classes_)\n\nprint(\"Classes:\", NUM_CLASSES)\n\n# ------------------------\n# Dataset\n# ------------------------\nclass IMUWindowDataset(Dataset):\n    def __init__(self, df):\n        self.samples = []\n\n        for seq_id, seq in df.groupby(\"sequence_id\"):\n            seq = seq.reset_index(drop=True)\n\n            if \"Gesture\" not in seq[\"phase\"].values:\n                continue\n\n            center_idx = seq.index[seq[\"phase\"] == \"Gesture\"][0]\n\n            start = center_idx - CENTER_OFFSET\n            end = start + WINDOW_LEN\n\n            if start < 0 or end > len(seq):\n                continue\n\n            window = seq.iloc[start:end]\n\n            x = window[[\"acc_x\", \"acc_y\", \"acc_z\"]].values.astype(np.float32)\n            y = window[\"gesture_int\"].iloc[CENTER_OFFSET]\n\n            if np.isnan(x).any():\n                continue\n\n            self.samples.append((x, y))\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, idx):\n        x, y = self.samples[idx]\n        return torch.tensor(x), torch.tensor(y)\n\n# ------------------------\n# Train / Val split (by subject if exists)\n# ------------------------\nif \"subject\" in df.columns:\n    subjects = df[\"subject\"].unique()\n    np.random.shuffle(subjects)\n    split = int(0.8 * len(subjects))\n    train_subj = set(subjects[:split])\n\n    train_df = df[df[\"subject\"].isin(train_subj)]\n    val_df   = df[~df[\"subject\"].isin(train_subj)]\nelse:\n    train_df = df.sample(frac=0.8, random_state=42)\n    val_df = df.drop(train_df.index)\n\ntrain_ds = IMUWindowDataset(train_df)\nval_ds   = IMUWindowDataset(val_df)\n\nprint(\"Train windows:\", len(train_ds))\nprint(\"Val windows:\", len(val_ds))\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\nval_loader   = DataLoader(val_ds, batch_size=BATCH_SIZE)\n\n# ------------------------\n# Model: DeepConvLSTM\n# ------------------------\nclass DeepConvLSTM(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n\n        self.conv = nn.Sequential(\n            nn.Conv1d(3, 64, 5, padding=2),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n\n            nn.Conv1d(64, 64, 5, padding=2),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n\n            nn.Conv1d(64, 128, 5, padding=2),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n\n            nn.Conv1d(128, 128, 5, padding=2),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n        )\n\n        self.lstm = nn.LSTM(\n            input_size=128,\n            hidden_size=128,\n            num_layers=2,\n            batch_first=True,\n            bidirectional=True,\n            dropout=0.5\n        )\n\n        self.fc = nn.Sequential(\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(128, num_classes)\n        )\n\n    def forward(self, x):\n        x = x.transpose(1, 2)          # (B, C, T)\n        x = self.conv(x)\n        x = x.transpose(1, 2)          # (B, T, C)\n\n        x, _ = self.lstm(x)\n        x = x.mean(dim=1)\n\n        return self.fc(x)\n\n# ------------------------\n# Training utils\n# ------------------------\ndef topk_acc(logits, y, k=3):\n    return (logits.topk(k, dim=1).indices == y.unsqueeze(1)).any(dim=1).float().mean().item()\n\ndef train_epoch(model, loader):\n    model.train()\n    acc, loss_sum = 0, 0\n\n    for x, y in loader:\n        x, y = x.to(DEVICE), y.to(DEVICE)\n\n        opt.zero_grad()\n        out = model(x)\n        loss = loss_fn(out, y)\n        loss.backward()\n        opt.step()\n\n        acc += (out.argmax(1) == y).float().mean().item()\n        loss_sum += loss.item()\n\n    return loss_sum / len(loader), acc / len(loader)\n\n@torch.no_grad()\ndef eval_epoch(model, loader):\n    model.eval()\n    acc, top3, loss_sum = 0, 0, 0\n    preds, labels = [], []\n\n    for x, y in loader:\n        x, y = x.to(DEVICE), y.to(DEVICE)\n        out = model(x)\n\n        loss = loss_fn(out, y)\n        loss_sum += loss.item()\n\n        acc += (out.argmax(1) == y).float().mean().item()\n        top3 += topk_acc(out, y, 3)\n\n        preds.append(out.argmax(1).cpu().numpy())\n        labels.append(y.cpu().numpy())\n\n    return (\n        loss_sum / len(loader),\n        acc / len(loader),\n        top3 / len(loader),\n        np.concatenate(preds),\n        np.concatenate(labels),\n    )\n\n# ------------------------\n# Train\n# ------------------------\nmodel = DeepConvLSTM(NUM_CLASSES).to(DEVICE)\nopt = torch.optim.Adam(model.parameters(), lr=LR)\nloss_fn = nn.CrossEntropyLoss()\n\nbest_acc = 0\n\nfor epoch in range(1, EPOCHS + 1):\n    tr_loss, tr_acc = train_epoch(model, train_loader)\n    val_loss, val_acc, val_top3, preds, labels = eval_epoch(model, val_loader)\n\n    print(\n        f\"Epoch {epoch:02d} | \"\n        f\"Train Acc {tr_acc:.3f} | \"\n        f\"Val Acc {val_acc:.3f} | \"\n        f\"Top-3 {val_top3:.3f}\"\n    )\n\n    if val_acc > best_acc:\n        best_acc = val_acc\n        torch.save(model.state_dict(), \"best_model.pt\")\n\n# ------------------------\n# Confusion Matrix\n# ------------------------\ncm = confusion_matrix(labels, preds)\nplt.figure(figsize=(8, 6))\nsns.heatmap(cm, cmap=\"Blues\", xticklabels=le.classes_, yticklabels=le.classes_)\nplt.title(\"Confusion Matrix\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.show()\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-02T17:49:34.931943Z","iopub.execute_input":"2026-02-02T17:49:34.9323Z","iopub.status.idle":"2026-02-02T17:51:29.561458Z","shell.execute_reply.started":"2026-02-02T17:49:34.932272Z","shell.execute_reply":"2026-02-02T17:51:29.560711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nimport numpy as np\n\n# -------------------------\n# Dataset\n# -------------------------\nclass GestureDataset(Dataset):\n    def __init__(self, df, imu_cols, thm_cols, tof_cols, label_col):\n        self.df = df\n        self.imu_cols = imu_cols\n        self.thm_cols = thm_cols\n        self.tof_cols = tof_cols\n        self.labels = df[label_col].values\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        imu = row[self.imu_cols].values.astype(np.float32)\n        thm = row[self.thm_cols].values.astype(np.float32)\n        tof = row[self.tof_cols].values.astype(np.float32)\n\n        # Replace missing TOF with 500 and normalize\n        tof[tof == -1] = 500\n        tof = tof / 500.0\n\n        x = np.concatenate([imu, thm, tof], axis=-1)\n        y = self.labels[idx]\n        return torch.tensor(x, dtype=torch.float32), torch.tensor(y, dtype=torch.long)\n\n# -------------------------\n# Model\n# -------------------------\nclass ResidualBlock1D(nn.Module):\n    def __init__(self, in_ch, out_ch, kernel=3, drop=0.3):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_ch, out_ch, kernel, padding=kernel//2)\n        self.bn1 = nn.BatchNorm1d(out_ch)\n        self.conv2 = nn.Conv1d(out_ch, out_ch, kernel, padding=kernel//2)\n        self.bn2 = nn.BatchNorm1d(out_ch)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(drop)\n        self.pool = nn.MaxPool1d(2)\n        self.shortcut = nn.Conv1d(in_ch, out_ch, 1) if in_ch != out_ch else nn.Identity()\n\n    def forward(self, x):\n        identity = self.shortcut(x)\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out += identity\n        out = self.relu(out)\n        out = self.pool(out)\n        out = self.dropout(out)\n        return out\n\nclass GestureModel(nn.Module):\n    def __init__(self, imu_ch, thm_ch, tof_ch, n_classes):\n        super().__init__()\n        # Branches\n        self.imu_branch = nn.Sequential(\n            ResidualBlock1D(imu_ch, 128),\n            ResidualBlock1D(128, 256)\n        )\n        self.thm_branch = nn.Sequential(\n            ResidualBlock1D(thm_ch, 64),\n            ResidualBlock1D(64, 128)\n        )\n        self.tof_branch = nn.Sequential(\n            ResidualBlock1D(tof_ch, 128),\n            ResidualBlock1D(128, 256)\n        )\n\n        # BiLSTM over concatenated features\n        self.bilstm = nn.LSTM(256+256+128, 512, batch_first=True, bidirectional=True)\n        self.attn_fc = nn.Linear(1024, 512)\n        self.fc = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, n_classes)\n        )\n\n    def forward(self, x):\n        # x: [B, T, C]\n        imu_ch = 12\n        thm_ch = 5\n        tof_ch = x.shape[-1] - imu_ch - thm_ch\n\n        imu = x[:, :, :imu_ch].transpose(1,2)\n        thm = x[:, :, imu_ch:imu_ch+thm_ch].transpose(1,2)\n        tof = x[:, :, imu_ch+thm_ch:].transpose(1,2)\n\n        imu_feat = self.imu_branch(imu).transpose(1,2)\n        thm_feat = self.thm_branch(thm).transpose(1,2)\n        tof_feat = self.tof_branch(tof).transpose(1,2)\n\n        merged = torch.cat([imu_feat, thm_feat, tof_feat], dim=2)\n\n        lstm_out, _ = self.bilstm(merged)\n        attn_weights = F.softmax(self.attn_fc(lstm_out), dim=1)\n        context = torch.sum(attn_weights * lstm_out, dim=1)\n        out = self.fc(context)\n        return out\n\n# -------------------------\n# Training Loop\n# -------------------------\ndef train_loop(model, loader, optimizer, criterion, device):\n    model.train()\n    total, correct = 0, 0\n    for x, y in loader:\n        x, y = x.to(device), y.to(device)\n        optimizer.zero_grad()\n        out = model(x)\n        loss = criterion(out, y)\n        loss.backward()\n        optimizer.step()\n        total += y.size(0)\n        correct += (out.argmax(1) == y).sum().item()\n    return correct/total\n\ndef val_loop(model, loader, criterion, device):\n    model.eval()\n    total, correct = 0, 0\n    with torch.no_grad():\n        for x, y in loader:\n            x, y = x.to(device), y.to(device)\n            out = model(x)\n            total += y.size(0)\n            correct += (out.argmax(1) == y).sum().item()\n    return correct/total\n\n# -------------------------\n# Usage Example\n# -------------------------\nif __name__==\"__main__\":\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n    imu_cols = [c for c in df.columns if \"acc\" in c or \"rot\" in c]\n    thm_cols = [c for c in df.columns if \"thm\" in c]\n    tof_cols = [c for c in df.columns if \"tof\" in c]\n\n    dataset = GestureDataset(df, imu_cols, thm_cols, tof_cols, \"gesture\")\n    loader = DataLoader(dataset, batch_size=64, shuffle=True)\n\n    model = GestureModel(len(imu_cols), len(thm_cols), len(tof_cols), n_classes=20)\n    model.to(device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n    criterion = nn.CrossEntropyLoss()\n\n    for epoch in range(10):\n        train_acc = train_loop(model, loader, optimizer, criterion, device)\n        val_acc = val_loop(model, loader, criterion, device)\n        print(f\"Epoch {epoch+1} | Train Acc {train_acc:.3f} | Val Acc {val_acc:.3f}\")\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-02T17:51:29.563612Z","iopub.execute_input":"2026-02-02T17:51:29.563846Z","iopub.status.idle":"2026-02-02T17:51:47.65453Z","shell.execute_reply.started":"2026-02-02T17:51:29.563829Z","shell.execute_reply":"2026-02-02T17:51:47.653471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================\n# CONFIGURATION\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nBATCH_SIZE = 64  # Original was good\nWINDOW_BEFORE = 30  # Original values\nWINDOW_AFTER = 30\nWINDOW_LEN = WINDOW_BEFORE + WINDOW_AFTER + 1\nEPOCHS = 35  # Increased slightly\nLR = 1e-3  # Original was good\nTOP_K = 3\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint(f\"Using device: {DEVICE}\")\nprint(f\"Window length: {WINDOW_LEN}\")\n\n# ============================================================\n# MINIMAL PREPROCESSING (KEEP IT SIMPLE)\n# ============================================================\n\ndef simple_quaternion_normalization(rot_data: np.ndarray) -> np.ndarray:\n    \"\"\"Fast quaternion normalization - keep it simple\"\"\"\n    rot_norm = rot_data.copy()\n    norms = np.linalg.norm(rot_data, axis=1, keepdims=True)\n    norms[norms < 1e-6] = 1.0\n    return rot_norm / norms\n\ndef extract_simple_features(acc, rot):\n    \"\"\"Original approach worked well - keep it\"\"\"\n    # 1. Normalize acceleration (per window)\n    acc_norm = (acc - acc.mean(0)) / (acc.std(0) + 1e-6)\n    \n    # 2. Normalize quaternions\n    rot_norm = simple_quaternion_normalization(rot)\n    \n    # 3. Add magnitude features (NEW - simple but effective)\n    acc_magnitude = np.linalg.norm(acc_norm, axis=1, keepdims=True)\n    rot_magnitude = np.linalg.norm(rot_norm, axis=1, keepdims=True)\n    \n    # 4. Add simple temporal difference (NEW - captures motion)\n    if len(acc_norm) > 1:\n        acc_diff = np.diff(acc_norm, axis=0, prepend=acc_norm[0:1])\n        rot_diff = np.diff(rot_norm, axis=0, prepend=rot_norm[0:1])\n    else:\n        acc_diff = np.zeros_like(acc_norm)\n        rot_diff = np.zeros_like(rot_norm)\n    \n    # Total: 7 (original) + 2 (magnitude) + 10 (differences) = 19 features\n    features = np.concatenate([\n        acc_norm,        # 3 features\n        rot_norm,        # 4 features\n        acc_magnitude,   # 1 feature\n        rot_magnitude,   # 1 feature\n        acc_diff,        # 3 features\n        rot_diff         # 7 features (4 rot + 3 acc diff for rot?)\n    ], axis=1)\n    \n    # Actually rot_diff is 4, so total is 3+4+1+1+3+4 = 16 features\n    return features[:, :16]  # Ensure consistent shape\n\n# ============================================================\n# DATASET - SIMILAR TO ORIGINAL BUT BETTER\n# ============================================================\n\nclass ImprovedIMUDataset(Dataset):\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n    \n    def __init__(self, csv_path, use_advanced_features=False):\n        df = pd.read_csv(csv_path)\n        \n        # Encode labels\n        df[\"gesture\"] = df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        df[\"gesture_id\"] = self.le.fit_transform(df[\"gesture\"])\n        self.classes = self.le.classes_\n        \n        self.windows = []\n        self.use_advanced_features = use_advanced_features\n        \n        print(f\"Processing {len(df['sequence_id'].unique())} sequences...\")\n        \n        sequence_count = 0\n        for seq_id, seq in df.groupby(\"sequence_id\"):\n            sequence_count += 1\n            # if sequence_count % 100 == 0:\n            #     print(f\"  Processed {sequence_count} sequences...\")\n                \n            seq = seq.reset_index(drop=True).copy()\n            \n            # Fill NaN values with 0 (simple approach)\n            seq[self.ACC_COLS] = seq[self.ACC_COLS].fillna(0)\n            seq[self.ROT_COLS] = seq[self.ROT_COLS].fillna(0)\n            \n            # Find ALL gesture points in this sequence\n            gesture_indices = seq[seq[\"phase\"] == \"Gesture\"].index.tolist()\n            \n            for center_idx in gesture_indices[:2]:  # Use up to 2 gestures per sequence\n                start_idx = center_idx - WINDOW_BEFORE\n                end_idx = center_idx + WINDOW_AFTER + 1\n                \n                # Adjust boundaries\n                start_idx = max(0, start_idx)\n                end_idx = min(len(seq), end_idx)\n                \n                # Skip if window is too small\n                if end_idx - start_idx < 20:\n                    continue\n                \n                # Extract data\n                acc_data = seq.iloc[start_idx:end_idx][self.ACC_COLS].values.astype(np.float32)\n                rot_data = seq.iloc[start_idx:end_idx][self.ROT_COLS].values.astype(np.float32)\n                \n                # Skip empty windows\n                if len(acc_data) == 0:\n                    continue\n                \n                # Extract features\n                if self.use_advanced_features:\n                    features = extract_simple_features(acc_data, rot_data)\n                else:\n                    # Original approach\n                    acc_norm = (acc_data - acc_data.mean(0)) / (acc_data.std(0) + 1e-6)\n                    rot_norm = simple_quaternion_normalization(rot_data)\n                    features = np.concatenate([acc_norm, rot_norm], axis=1)\n                \n                # Pad or truncate to exact window length\n                if len(features) < WINDOW_LEN:\n                    # Pad with zeros\n                    pad_len = WINDOW_LEN - len(features)\n                    features = np.pad(features, ((0, pad_len), (0, 0)), mode='constant')\n                else:\n                    # Truncate to window length\n                    features = features[:WINDOW_LEN]\n                \n                # Get label\n                label = seq.loc[center_idx, \"gesture_id\"]\n                \n                # Store as (features, label)\n                self.windows.append((features.T, label))\n        \n        print(f\"\\nCreated {len(self.windows)} windows\")\n        print(f\"Feature dimension: {self.windows[0][0].shape if self.windows else 'No windows'}\")\n    \n    def __len__(self):\n        return len(self.windows)\n    \n    def __getitem__(self, idx):\n        features, label = self.windows[idx]\n        return (\n            torch.tensor(features, dtype=torch.float32),\n            torch.tensor(label, dtype=torch.long),\n        )\n\n# ============================================================\n# IMPROVED INCEPTIONTIME ARCHITECTURE\n# ============================================================\n\nclass BasicInceptionModule(nn.Module):\n    \"\"\"Basic Inception module (no residual connections)\"\"\"\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        # Branch 1: 1x1 convolution\n        self.branch1 = nn.Conv1d(in_channels, out_channels, kernel_size=1)\n        \n        # Branch 2: 1x1 -> 3x3 convolution\n        self.branch2 = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, kernel_size=1),\n            nn.Conv1d(out_channels, out_channels, kernel_size=3, padding=1),\n        )\n        \n        # Branch 3: 1x1 -> 5x5 convolution\n        self.branch3 = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, kernel_size=1),\n            nn.Conv1d(out_channels, out_channels, kernel_size=5, padding=2),\n        )\n        \n        # Branch 4: 3x3 max pool -> 1x1 convolution\n        self.branch4 = nn.Sequential(\n            nn.MaxPool1d(kernel_size=3, stride=1, padding=1),\n            nn.Conv1d(in_channels, out_channels, kernel_size=1),\n        )\n        \n        # Batch normalization and activation (applied after concatenation)\n        self.bn = nn.BatchNorm1d(out_channels * 4)\n        self.relu = nn.ReLU()\n        \n    def forward(self, x):\n        # Process through all branches\n        out1 = self.branch1(x)\n        out2 = self.branch2(x)\n        out3 = self.branch3(x)\n        out4 = self.branch4(x)\n        \n        # Concatenate\n        out = torch.cat([out1, out2, out3, out4], dim=1)\n        \n        # Apply batch norm and activation\n        out = self.relu(self.bn(out))\n        \n        return out\n\nclass BasicInceptionTime(nn.Module):\n    \"\"\"Basic InceptionTime model (original architecture)\"\"\"\n    def __init__(self, in_channels, num_classes):\n        super().__init__()\n        \n        # Three basic Inception blocks (original uses 3)\n        self.inception1 = BasicInceptionModule(in_channels, 32)\n        self.inception2 = BasicInceptionModule(32 * 4, 32)\n        self.inception3 = BasicInceptionModule(32 * 4, 32)\n        \n        # Global pooling\n        self.global_pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Single fully connected layer (original architecture)\n        self.fc = nn.Linear(32 * 4, num_classes)\n        \n    def forward(self, x):\n        # Through inception blocks\n        x = self.inception1(x)\n        x = self.inception2(x)\n        x = self.inception3(x)\n        \n        # Global pooling\n        x = self.global_pool(x)\n        x = x.view(x.size(0), -1)\n        \n        # Classification\n        x = self.fc(x)\n        \n        return x\n\n# ============================================================\n# EVEN SIMPLER VERSION (Closest to original paper)\n# ============================================================\n\nclass PaperInceptionModule(nn.Module):\n    \"\"\"Inception module as described in the original InceptionTime paper\"\"\"\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        # 40 filters per branch (as in paper)\n        filters_per_branch = out_channels\n        \n        self.bottleneck = nn.Conv1d(in_channels, filters_per_branch, kernel_size=1)\n        \n        self.conv1 = nn.Conv1d(filters_per_branch, filters_per_branch, kernel_size=1)\n        self.conv3 = nn.Conv1d(filters_per_branch, filters_per_branch, kernel_size=3, padding=1)\n        self.conv5 = nn.Conv1d(filters_per_branch, filters_per_branch, kernel_size=5, padding=2)\n        \n        self.maxpool = nn.MaxPool1d(kernel_size=3, stride=1, padding=1)\n        self.convpool = nn.Conv1d(in_channels, filters_per_branch, kernel_size=1)\n        \n        self.bn = nn.BatchNorm1d(filters_per_branch * 4)\n        self.relu = nn.ReLU()\n        \n    def forward(self, x):\n        # Bottleneck layer\n        bottleneck_out = self.bottleneck(x)\n        \n        # Three convolutions\n        out1 = self.conv1(bottleneck_out)\n        out2 = self.conv3(bottleneck_out)\n        out3 = self.conv5(bottleneck_out)\n        \n        # Max pooling branch\n        out4 = self.maxpool(x)\n        out4 = self.convpool(out4)\n        \n        # Concatenate and activate\n        out = torch.cat([out1, out2, out3, out4], dim=1)\n        out = self.relu(self.bn(out))\n        \n        return out\n\nclass PaperInceptionTime(nn.Module):\n    \"\"\"InceptionTime as described in the original paper\"\"\"\n    def __init__(self, in_channels, num_classes, num_blocks=3, filters=32):\n        super().__init__()\n        \n        self.blocks = nn.ModuleList()\n        \n        # First block\n        self.blocks.append(PaperInceptionModule(in_channels, filters))\n        \n        # Additional blocks\n        for _ in range(1, num_blocks):\n            self.blocks.append(PaperInceptionModule(filters * 4, filters))\n        \n        # Global pooling\n        self.global_pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Fully connected layer\n        self.fc = nn.Linear(filters * 4, num_classes)\n        \n    def forward(self, x):\n        for block in self.blocks:\n            x = block(x)\n        \n        x = self.global_pool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n        \n        return x\n\n# ============================================================\n# MINIMALIST VERSION (Closest to your original code)\n# ============================================================\n\nclass MinimalInceptionBlock(nn.Module):\n    \"\"\"Minimal Inception block matching your original code structure\"\"\"\n    def __init__(self, in_ch, out_ch):\n        super().__init__()\n\n        self.branch1 = nn.Conv1d(in_ch, out_ch, 1)\n\n        self.branch3 = nn.Sequential(\n            nn.Conv1d(in_ch, out_ch, 1),\n            nn.Conv1d(out_ch, out_ch, 3, padding=1),\n        )\n\n        self.branch5 = nn.Sequential(\n            nn.Conv1d(in_ch, out_ch, 1),\n            nn.Conv1d(out_ch, out_ch, 5, padding=2),\n        )\n\n        self.pool = nn.Sequential(\n            nn.MaxPool1d(3, stride=1, padding=1),\n            nn.Conv1d(in_ch, out_ch, 1),\n        )\n\n        self.bn = nn.BatchNorm1d(out_ch * 4)\n        self.relu = nn.ReLU()\n\n    def forward(self, x):\n        x = torch.cat([\n            self.branch1(x),\n            self.branch3(x),\n            self.branch5(x),\n            self.pool(x)\n        ], dim=1)\n        return self.relu(self.bn(x))\n\nclass MinimalInceptionTime(nn.Module):\n    \"\"\"Minimal InceptionTime matching your original code exactly\"\"\"\n    def __init__(self, in_channels, num_classes):\n        super().__init__()\n\n        self.block1 = MinimalInceptionBlock(in_channels, 32)\n        self.block2 = MinimalInceptionBlock(32 * 4, 32)\n        self.block3 = MinimalInceptionBlock(32 * 4, 32)\n\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        self.fc = nn.Linear(32 * 4, num_classes)\n\n    def forward(self, x):\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.pool(x).squeeze(-1)\n        return self.fc(x)\nclass InceptionModule(nn.Module):\n    \"\"\"Improved Inception module with better feature extraction\"\"\"\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        \n        # Branch 1: 1x1 convolution\n        self.branch1 = nn.Conv1d(in_channels, out_channels, kernel_size=1)\n        \n        # Branch 2: 1x1 -> 3x3 convolution\n        self.branch2 = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, kernel_size=1),\n            nn.Conv1d(out_channels, out_channels, kernel_size=3, padding=1),\n        )\n        \n        # Branch 3: 1x1 -> 5x5 convolution\n        self.branch3 = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, kernel_size=1),\n            nn.Conv1d(out_channels, out_channels, kernel_size=5, padding=2),\n        )\n        \n        # Branch 4: 3x3 max pool -> 1x1 convolution\n        self.branch4 = nn.Sequential(\n            nn.MaxPool1d(kernel_size=3, stride=1, padding=1),\n            nn.Conv1d(in_channels, out_channels, kernel_size=1),\n        )\n        \n        # Batch normalization and activation\n        self.bn = nn.BatchNorm1d(out_channels * 4)\n        self.activation = nn.ReLU()\n        \n        # Residual connection (NEW - helps with deeper networks)\n        self.residual = None\n        if in_channels != out_channels * 4:\n            self.residual = nn.Conv1d(in_channels, out_channels * 4, kernel_size=1)\n        \n    def forward(self, x):\n        # Store residual\n        residual = x\n        \n        # Process through all branches\n        out1 = self.branch1(x)\n        out2 = self.branch2(x)\n        out3 = self.branch3(x)\n        out4 = self.branch4(x)\n        \n        # Concatenate\n        out = torch.cat([out1, out2, out3, out4], dim=1)\n        \n        # Apply residual if needed\n        if self.residual is not None:\n            residual = self.residual(residual)\n        \n        # Add residual and apply batch norm + activation\n        out = out + residual\n        out = self.activation(self.bn(out))\n        \n        return out\n\nclass ImprovedInceptionTime(nn.Module):\n    \"\"\"Improved InceptionTime with better depth and feature usage\"\"\"\n    def __init__(self, in_channels, num_classes, bottleneck=True):\n        super().__init__()\n        \n        # Initial convolution to increase channels\n        self.initial_conv = nn.Sequential(\n            nn.Conv1d(in_channels, 32, kernel_size=7, padding=3),\n            nn.BatchNorm1d(32),\n            nn.ReLU(),\n        )\n        \n        # Inception modules with increasing complexity\n        self.inception1 = InceptionModule(32, 32)  # Output: 128 channels\n        self.inception2 = InceptionModule(128, 32) # Output: 128 channels\n        self.inception3 = InceptionModule(128, 32) # Output: 128 channels\n        self.inception4 = InceptionModule(128, 32) # Output: 128 channels\n        \n        # Global pooling\n        self.global_pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Optional bottleneck layer (reduces overfitting)\n        self.bottleneck = bottleneck\n        if bottleneck:\n            self.fc1 = nn.Linear(128, 64)\n            self.bn1 = nn.BatchNorm1d(64)\n            self.dropout = nn.Dropout(0.3)\n            self.fc2 = nn.Linear(64, num_classes)\n        else:\n            self.fc = nn.Linear(128, num_classes)\n        \n    def forward(self, x):\n        # Initial feature extraction\n        x = self.initial_conv(x)\n        \n        # Stack of inception modules\n        x = self.inception1(x)\n        x = self.inception2(x)\n        x = self.inception3(x)\n        x = self.inception4(x)\n        \n        # Global pooling\n        x = self.global_pool(x)\n        x = x.view(x.size(0), -1)\n        \n        # Classification\n        if self.bottleneck:\n            x = F.relu(self.bn1(self.fc1(x)))\n            x = self.dropout(x)\n            x = self.fc2(x)\n        else:\n            x = self.fc(x)\n        \n        return x\n\n# ============================================================\n# TRAINING FUNCTIONS WITH IMPROVEMENTS\n# ============================================================\n\ndef train_model(model, train_loader, val_loader, num_epochs=EPOCHS, learning_rate=LR):\n    \"\"\"Train with improvements\"\"\"\n    device = DEVICE\n    \n    # Loss and optimizer\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=1e-4)\n    \n    # Learning rate scheduler (NEW - helps converge better)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n    \n    # Track metrics\n    history = {\n        'train_loss': [], 'train_acc': [],\n        'val_loss': [], 'val_acc': []\n    }\n    \n    best_val_acc = 0\n    best_model_state = None\n    \n    print(\"Starting training...\")\n    print(\"-\" * 80)\n    \n    for epoch in range(num_epochs):\n        # Training phase\n        model.train()\n        train_loss = 0\n        train_correct = 0\n        train_total = 0\n        \n        for batch_idx, (data, target) in enumerate(train_loader):\n            data, target = data.to(device), target.to(device)\n            \n            optimizer.zero_grad()\n            output = model(data)\n            loss = criterion(output, target)\n            \n            # Add L2 regularization manually for more control\n            l2_lambda = 0.0001\n            l2_norm = sum(p.pow(2.0).sum() for p in model.parameters())\n            loss = loss + l2_lambda * l2_norm\n            \n            loss.backward()\n            \n            # Gradient clipping (prevents explosions)\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            \n            optimizer.step()\n            \n            # Statistics\n            train_loss += loss.item()\n            pred = output.argmax(dim=1)\n            train_correct += (pred == target).sum().item()\n            train_total += target.size(0)\n            \n            # if batch_idx % 20 == 0:\n            #     print(f\"  Batch {batch_idx}/{len(train_loader)} - Loss: {loss.item():.4f}\")\n        \n        train_loss = train_loss / len(train_loader)\n        train_acc = train_correct / train_total\n        \n        # Validation phase\n        model.eval()\n        val_loss = 0\n        val_correct = 0\n        val_total = 0\n        \n        with torch.no_grad():\n            for data, target in val_loader:\n                data, target = data.to(device), target.to(device)\n                output = model(data)\n                loss = criterion(output, target)\n                \n                val_loss += loss.item()\n                pred = output.argmax(dim=1)\n                val_correct += (pred == target).sum().item()\n                val_total += target.size(0)\n        \n        val_loss = val_loss / len(val_loader)\n        val_acc = val_correct / val_total\n        \n        # Update learning rate\n        scheduler.step()\n        \n        # Store history\n        history['train_loss'].append(train_loss)\n        history['train_acc'].append(train_acc)\n        history['val_loss'].append(val_loss)\n        history['val_acc'].append(val_acc)\n        \n        # Save best model\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_model_state = model.state_dict().copy()\n            torch.save(best_model_state, \"/kaggle/working/best_inceptiontime_model.pth\")\n        \n        # Print progress\n        print(f\"Epoch {epoch+1:03d}/{num_epochs} | \"\n              f\"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.4f} | \"\n              f\"Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.4f} | \"\n              f\"LR: {scheduler.get_last_lr()[0]:.6f}\")\n    \n    # Load best model\n    if best_model_state is not None:\n        model.load_state_dict(best_model_state)\n    \n    print(f\"\\nBest validation accuracy: {best_val_acc:.4f}\")\n    \n    return model, history, best_val_acc\n    \nclass RawTestDataset(Dataset):\n    \"\"\"\n    Test dataset that returns pandas DataFrames grouped by sequence_id\n    \"\"\"\n    def __init__(self, csv_path):\n        df = pd.read_csv(csv_path)\n        \n        # Group by sequence_id and store each as DataFrame\n        self.sequences = []  # List of DataFrames\n        self.sequence_ids = []  # List of sequence IDs\n        \n        for seq_id, seq_df in df.groupby(\"sequence_id\"):\n            # Reset index for this sequence\n            seq_df = seq_df.reset_index(drop=True)\n            \n            self.sequences.append(seq_df)\n            self.sequence_ids.append(seq_id)\n        \n        print(f\"Loaded {len(self.sequences)} sequences from {csv_path}\")\n    \n    def __len__(self):\n        return len(self.sequences)\n    \n    def __getitem__(self, idx):\n        # Returns: (sequence_data_as_DataFrame, sequence_id)\n        return self.sequences[idx], self.sequence_ids[idx]\n\n# Custom collate function for DataFrames\ndef df_collate_fn(batch):\n    \"\"\"\n    Custom collate function to handle pandas DataFrames\n    \"\"\"\n    # batch is a list of tuples: [(df1, id1), (df2, id2), ...]\n    \n    # Separate DataFrames and IDs\n    data = [item[0] for item in batch]  # List of DataFrames\n    seq_ids = [item[1] for item in batch]  # List of sequence IDs\n    \n    return data, seq_ids\ndef prepare_window(sequence_df):\n    def simple_quaternion_normalization(rot_data: np.ndarray) -> np.ndarray:\n        \"\"\"Fast quaternion normalization\"\"\"\n        rot_norm = rot_data.copy()\n        norms = np.linalg.norm(rot_data, axis=1, keepdims=True)\n        norms[norms < 1e-6] = 1.0\n        return rot_norm / norms\n    \n    def extract_features(acc, rot):\n        \"\"\"Extract features (must match training)\"\"\"\n        # Handle empty arrays\n        if len(acc) == 0:\n            return np.zeros((WINDOW_LEN, 16), dtype=np.float32)\n        \n        # Normalize acceleration\n        acc_mean = acc.mean(0)\n        acc_std = acc.std(0) + 1e-6\n        acc_norm = (acc - acc_mean) / acc_std\n        \n        # Normalize quaternions\n        rot_norm = simple_quaternion_normalization(rot)\n        \n        # Add magnitude features\n        acc_magnitude = np.linalg.norm(acc_norm, axis=1, keepdims=True)\n        rot_magnitude = np.linalg.norm(rot_norm, axis=1, keepdims=True)\n        \n        # Add temporal differences\n        if len(acc_norm) > 1:\n            acc_diff = np.diff(acc_norm, axis=0, prepend=acc_norm[0:1])\n            rot_diff = np.diff(rot_norm, axis=0, prepend=rot_norm[0:1])\n        else:\n            acc_diff = np.zeros_like(acc_norm)\n            rot_diff = np.zeros_like(rot_norm)\n        \n        # Combine features (16 total)\n        features = np.concatenate([\n            acc_norm,        # 3\n            rot_norm,        # 4\n            acc_magnitude,   # 1\n            rot_magnitude,   # 1\n            acc_diff,        # 3\n            rot_diff         # 4\n        ], axis=1)\n        \n        return features[:, :16]  # Ensure consistent shape\n    def find_activity_window(acc_data):\n        \"\"\"\n        Find the most active window in the sequence when phase is not available.\n        Uses variance of acceleration magnitude to find interesting parts.\n        \"\"\"\n        if len(acc_data) < WINDOW_LEN:\n            # If sequence is shorter than window, use the whole sequence\n            return 0, len(acc_data)\n        \n        # Calculate acceleration magnitude\n        acc_magnitude = np.linalg.norm(acc_data, axis=1)\n        \n        # Find window with maximum variance (most activity)\n        max_variance = -1\n        best_start = 0\n        \n        # Slide window through sequence\n        for start in range(0, len(acc_data) - WINDOW_LEN + 1, WINDOW_LEN // 4):\n            end = start + WINDOW_LEN\n            window_magnitude = acc_magnitude[start:end]\n            variance = np.var(window_magnitude)\n            \n            if variance > max_variance:\n                max_variance = variance\n                best_start = start\n        \n        return best_start, best_start + WINDOW_LEN\n        \n    \"\"\"Prepare a window from sequence for prediction (NO PHASE COLUMN)\"\"\"\n    # Convert to pandas for easier processing\n    if hasattr(sequence_df, 'to_pandas'):\n        df = sequence_df.to_pandas()\n    else:\n        df = sequence_df\n    \n    # Get accelerometer and rotation data\n    acc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    rot_cols = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n    \n    # Check if phase column exists\n    has_phase = \"phase\" in df.columns\n    \n    if has_phase and \"Gesture\" in df[\"phase\"].values:\n        # If phase column exists and has \"Gesture\", use it\n        gesture_rows = df[df[\"phase\"] == \"Gesture\"]\n        center_idx = gesture_rows.index[0]\n    else:\n        # No phase column or no gesture phase - find most active window\n        acc_data = df[acc_cols].values.astype(np.float32)\n        start_idx, end_idx = self.find_activity_window(acc_data)\n        \n        # Extract window\n        acc_data = df.iloc[start_idx:end_idx][acc_cols].values.astype(np.float32)\n        rot_data = df.iloc[start_idx:end_idx][rot_cols].values.astype(np.float32)\n        \n        # Extract features and return\n        features = extract_features(acc_data, rot_data)\n        \n        # Pad if necessary\n        if len(features) < WINDOW_LEN:\n            pad_len = WINDOW_LEN - len(features)\n            features = np.pad(features, ((0, pad_len), (0, 0)), mode='constant')\n        \n        features = features[:WINDOW_LEN]\n        return features.T  # Shape: (16, WINDOW_LEN)\n    \n    # If we have a phase column with gesture, proceed as before\n    start_idx = max(0, center_idx - WINDOW_BEFORE)\n    end_idx = min(len(df), center_idx + WINDOW_AFTER + 1)\n    \n    acc_data = df.iloc[start_idx:end_idx][acc_cols].values.astype(np.float32)\n    rot_data = df.iloc[start_idx:end_idx][rot_cols].values.astype(np.float32)\n    \n    # Handle missing values\n    acc_data = np.nan_to_num(acc_data)\n    rot_data = np.nan_to_num(rot_data)\n    \n    # Extract features\n    features = extract_features(acc_data, rot_data)\n    \n    # Pad if necessary\n    if len(features) < WINDOW_LEN:\n        pad_len = WINDOW_LEN - len(features)\n        features = np.pad(features, ((0, pad_len), (0, 0)), mode='constant')\n    \n    # Trim to exact window length\n    features = features[:WINDOW_LEN]\n    \n    # Transpose for model (channels first)\n    return features.T  # Shape: (16, WINDOW_LEN)            \ndef evaluate_model(model, test_loader, dataset_classes):\n    \"\"\"Comprehensive evaluation\"\"\"\n    device = DEVICE\n    model.eval()\n    \n    all_preds = []\n    all_targets = []\n    all_probs = []\n    \n    with torch.no_grad():\n        for data, target in test_loader:\n            # features = prepare_window(data)\n            # features_tensor = torch.tensor(features, dtype=torch.float32).unsqueeze(0).to(DEVICE)\n            output = model(data)\n            probs = F.softmax(output, dim=1)\n            preds = output.argmax(dim=1)\n            \n            all_preds.extend(preds.cpu().numpy())\n            all_targets.extend(target.numpy())\n            all_probs.append(probs.cpu().numpy())\n    \n    all_preds = np.array(all_preds)\n    all_targets = np.array(all_targets)\n    all_probs = np.concatenate(all_probs, axis=0)\n    \n    # Calculate metrics\n    accuracy = (all_preds == all_targets).mean()\n    \n    # Top-K accuracy\n    topk_correct = 0\n    for i in range(len(all_targets)):\n        top_k_indices = np.argsort(all_probs[i])[-TOP_K:]\n        if all_targets[i] in top_k_indices:\n            topk_correct += 1\n    topk_acc = topk_correct / len(all_targets)\n    \n    print(f\"\\nEvaluation Results:\")\n    print(f\"  Accuracy: {accuracy:.4f}\")\n    print(f\"  Top-{TOP_K} Accuracy: {topk_acc:.4f}\")\n    \n    return all_preds, all_targets, all_probs, accuracy, topk_acc\n\n# ============================================================\n# MAIN EXECUTION\n# ============================================================\n\ndef main():\n    print(\"=\" * 80)\n    print(\"IMPROVED INCEPTIONTIME FOR GESTURE RECOGNITION\")\n    print(\"=\" * 80)\n    \n    # Load dataset with improved features\n    print(\"\\n1. Loading and preparing dataset...\")\n    dataset = ImprovedIMUDataset(CSV_PATH, use_advanced_features=True)\n    \n    if len(dataset) == 0:\n        print(\"ERROR: No data loaded!\")\n        return\n    \n    num_classes = len(dataset.classes)\n    in_channels = dataset.windows[0][0].shape[0]\n    \n    print(f\"   Samples: {len(dataset)}\")\n    print(f\"   Classes: {num_classes} ({dataset.classes})\")\n    print(f\"   Input shape: {dataset.windows[0][0].shape}\")\n    \n    # Split dataset (80% train, 10% val, 10% test)\n    train_size = int(0.8 * len(dataset))\n    val_size = int(0.1 * len(dataset))\n    test_size = len(dataset) - train_size - val_size\n    \n    train_dataset, val_dataset, test_dataset = random_split(\n        dataset, [train_size, val_size, test_size]\n    )\n    \n    # Create data loaders\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n    \n    # test_dataset = RawTestDataset(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n    # test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False,\n    # collate_fn=df_collate_fn)\n    # test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n    \n    print(f\"\\n2. Creating model...\")\n    print(f\"   Input channels: {in_channels}\")\n    print(f\"   Number of classes: {num_classes}\")\n    \n    # Try different model configurations\n    models_to_try = [\n        (\"Minimal_InceptionTime\", MinimalInceptionTime(in_channels, num_classes)),\n        (\"Basic_InceptionTime\", BasicInceptionTime(in_channels, num_classes)),\n        (\"Paper_InceptionTime\", PaperInceptionTime(in_channels, num_classes, num_blocks=3, filters=32)),\n        (\"Standard_InceptionTime\", ImprovedInceptionTime(in_channels, num_classes, bottleneck=False)),\n        (\"Improved_InceptionTime\", ImprovedInceptionTime(in_channels, num_classes, bottleneck=True)),\n    ]\n    \n    best_model = None\n    best_accuracy = 0\n    best_model_name = \"\"\n    \n    for model_name, model in models_to_try:\n        print(f\"\\n{'='*60}\")\n        print(f\"Training: {model_name}\")\n        print(f\"{'='*60}\")\n        \n        model = model.to(DEVICE)\n        \n        # Count parameters\n        total_params = sum(p.numel() for p in model.parameters())\n        trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n        print(f\"   Parameters: {total_params:,} total, {trainable_params:,} trainable\")\n        \n        # Train\n        trained_model, history, val_acc = train_model(\n            model, train_loader, val_loader, num_epochs=EPOCHS\n        )\n        \n        # Evaluate on test set\n        _, _, _, test_acc, test_topk = evaluate_model(trained_model, test_loader, dataset.classes)\n        \n        model_path = f'/kaggle/working/{model_name}_model.pth'\n        torch.save(model.state_dict(), model_path)\n        # Track best model\n        if test_acc > best_accuracy:\n            best_accuracy = test_acc\n            best_model = trained_model\n            best_model_name = model_name\n        \n        # Plot training history for this model\n        fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n        \n        axes[0].plot(history['train_loss'], label='Train Loss', linewidth=2)\n        axes[0].plot(history['val_loss'], label='Val Loss', linewidth=2)\n        axes[0].set_xlabel('Epoch')\n        axes[0].set_ylabel('Loss')\n        axes[0].set_title(f'{model_name} - Loss History')\n        axes[0].legend()\n        axes[0].grid(True, alpha=0.3)\n        \n        axes[1].plot(history['train_acc'], label='Train Acc', linewidth=2)\n        axes[1].plot(history['val_acc'], label='Val Acc', linewidth=2)\n        axes[1].set_xlabel('Epoch')\n        axes[1].set_ylabel('Accuracy')\n        axes[1].set_title(f'{model_name} - Accuracy History')\n        axes[1].legend()\n        axes[1].grid(True, alpha=0.3)\n        \n        plt.tight_layout()\n        plt.show()\n    \n    # Final evaluation with best model\n    print(f\"\\n{'='*80}\")\n    print(f\"BEST MODEL: {best_model_name}\")\n    print(f\"Test Accuracy: {best_accuracy:.4f}\")\n    print(f\"{'='*80}\")\n    \n    # Get predictions from best model\n    test_preds, test_targets, test_probs, final_acc, final_topk = evaluate_model(\n        best_model, test_loader, dataset.classes\n    )\n    \n    # Confusion matrix\n    cm = confusion_matrix(test_targets, test_preds)\n    \n    plt.figure(figsize=(12, 10))\n    sns.heatmap(cm, \n                xticklabels=dataset.classes, \n                yticklabels=dataset.classes,\n                cmap=\"Blues\", \n                fmt=\"d\",\n                annot=True,\n                cbar_kws={'label': 'Count'},\n                annot_kws={\"size\": 10})\n    plt.title(f\"Confusion Matrix - {best_model_name}\\nAccuracy: {final_acc:.3f}\", fontsize=14)\n    plt.xlabel(\"Predicted Label\", fontsize=12)\n    plt.ylabel(\"True Label\", fontsize=12)\n    plt.tight_layout()\n    plt.show()\n    \n    # Classification report\n    print(\"\\nClassification Report:\")\n    print(\"-\" * 60)\n    report = classification_report(test_targets, test_preds, \n                                   target_names=dataset.classes, digits=3)\n    print(report)\n    \n    # Save final model\n    torch.save(best_model.state_dict(), \"/kaggle/working/final_gesture_model.pth\")\n    print(f\"\\nModel saved to: /kaggle/working/final_gesture_model.pth\")\n    \n    return best_model, dataset\n\n# ============================================================\n# RUN THE CODE\n# ============================================================\n\nmodel, dataset = main()\n\n# Quick test with a few samples\nprint(\"\\n\" + \"=\"*80)\nprint(\"QUICK INFERENCE TEST\")\nprint(\"=\"*80)\n\n# Get a test sample\n# test_loader = DataLoader(dataset, batch_size=1, shuffle=True)\nmodel.eval()\n\nfor i, (data, target) in enumerate(test_loader):\n# for batch_data, batch_seq_ids in test_loader:\n#     for sequence_df, seq_id in zip(batch_data, batch_seq_ids):\n    if i >= 3:  # Test 3 samples\n        break\n    # features = prepare_window(data)\n    # features_tensor = torch.tensor(features, dtype=torch.float32).unsqueeze(0).to(DEVICE)\n    # Predict\n    with torch.no_grad():\n        outputs = model(data)\n        probs = F.softmax(outputs, dim=1)\n        pred_idx = outputs.argmax(dim=1).item()\n    \n    # Get prediction\n    prediction = dataset.classes_[pred_idx]\n    confidence = probs[0, pred_idx].item()\n    \n    print(f\"Sample {i+1}: True={true_label:10s} | Pred={pred_label:10s} | \"\n          f\"Confidence={confidence:.3f} | \"\n          f\"{'CORRECT' if pred.item() == target.item() else 'WRONG'}\")\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.655545Z","iopub.status.idle":"2026-02-02T17:51:47.655839Z","shell.execute_reply.started":"2026-02-02T17:51:47.655717Z","shell.execute_reply":"2026-02-02T17:51:47.655728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# HYBRID CNN + TRANSFORMER FUSED MODEL\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom scipy.spatial.transform import Rotation as R\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nBATCH_SIZE = 32\nWINDOW_BEFORE = 10\nWINDOW_AFTER = 10\nWINDOW_LEN = WINDOW_BEFORE + WINDOW_AFTER + 1\nD_MODEL = 64\nNHEAD = 4\nLAYERS = 2\nEPOCHS = 30\nLR = 3e-4\nTOP_K = 3\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device:\", DEVICE)\n\n# ============================================================\n# HELPER FUNCTIONS (integrate previous ones)\n# ============================================================\n\ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    # Same as your helper\n    rot_cleaned = rot_data.copy()\n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        if missing_count == 0:\n            norm = np.linalg.norm(row)\n            rot_cleaned[i] = row / norm if norm > 1e-8 else [1.0,0.0,0.0,0.0]\n        elif missing_count == 1:\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            sum_squares = np.sum(valid_values**2)\n            if sum_squares <= 1.0:\n                missing_value = np.sqrt(max(0,1.0-sum_squares))\n                if i>0 and not np.isnan(rot_cleaned[i-1,missing_idx]):\n                    if rot_cleaned[i-1,missing_idx]<0: missing_value=-missing_value\n                rot_cleaned[i,missing_idx]=missing_value\n                rot_cleaned[i,~np.isnan(row)] = valid_values\n            else: rot_cleaned[i]=[1.0,0.0,0.0,0.0]\n        else:\n            rot_cleaned[i]=[1.0,0.0,0.0,0.0]\n    return rot_cleaned\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    try:\n        rot_scipy = rot[:,[1,2,3,0]]\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        mask = norms<1e-8\n        rot_scipy[mask] = [0.0,0.0,0.0,1.0]\n        r = R.from_quat(rot_scipy)\n        return r.apply(acc)\n    except:\n        return acc.copy()\n\ndef remove_gravity_from_acc(acc_data, rot_data):\n    num_samples = acc_data.shape[0]\n    linear_accel = np.zeros_like(acc_data)\n    gravity_world = np.array([0,0,9.81])\n    for i in range(num_samples):\n        if np.all(np.isnan(rot_data[i])) or np.all(np.isclose(rot_data[i],0)):\n            linear_accel[i,:] = acc_data[i,:]\n            continue\n        try:\n            rotation = R.from_quat(rot_data[i])\n            gravity_sensor_frame = rotation.apply(gravity_world,inverse=True)\n            linear_accel[i,:] = acc_data[i,:]-gravity_sensor_frame\n        except:\n            linear_accel[i,:] = acc_data[i,:]\n    return linear_accel\n\ndef calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n    num_samples = rot_data.shape[0]\n    angular_vel = np.zeros((num_samples,3))\n    for i in range(num_samples-1):\n        q_t = rot_data[i]; q_t_plus = rot_data[i+1]\n        if np.any(np.isnan(q_t)) or np.any(np.isnan(q_t_plus)): continue\n        try:\n            r1 = R.from_quat(q_t); r2 = R.from_quat(q_t_plus)\n            delta_rot = r1.inv()*r2\n            angular_vel[i,:] = delta_rot.as_rotvec()/time_delta\n        except: continue\n    return angular_vel\n\ndef calculate_angular_distance(rot_data):\n    num_samples = rot_data.shape[0]\n    angular_dist = np.zeros(num_samples)\n    for i in range(num_samples-1):\n        q1=rot_data[i]; q2=rot_data[i+1]\n        if np.any(np.isnan(q1)) or np.any(np.isnan(q2)): continue\n        try:\n            r1=R.from_quat(q1); r2=R.from_quat(q2)\n            rel_rot = r1.inv()*r2\n            angular_dist[i]=np.linalg.norm(rel_rot.as_rotvec())\n        except: continue\n    return angular_dist\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass AccRotHybridDataset(Dataset):\n    ACC_COLS = [\"acc_x\",\"acc_y\",\"acc_z\"]\n    ROT_COLS = [\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]\n\n    def __init__(self,csv_path,window_before=10,window_after=10):\n        self.df=pd.read_csv(csv_path)\n        self.window_before = window_before\n        self.window_after = window_after\n        self.window_len = window_before+window_after+1\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        self.windows=[]\n\n        for _,seq in self.sequences:\n            seq = seq.reset_index(drop=True)\n            gesture_rows = seq[seq[\"phase\"]==\"Gesture\"]\n            if gesture_rows.empty: continue\n            center = gesture_rows.index[0]\n            start=center-self.window_before; end=center+self.window_after+1\n            start_c = max(0,start); end_c=min(len(seq),end)\n\n            acc = seq.iloc[start_c:end_c][self.ACC_COLS].values.astype(np.float32)\n            rot = seq.iloc[start_c:end_c][self.ROT_COLS].values.astype(np.float32)\n\n            # Preprocess rotation\n            rot = handle_quaternion_missing_values(rot)\n            norms = np.linalg.norm(rot,axis=1,keepdims=True); norms[norms<1e-6]=1.0\n            rot = rot/norms\n\n            # Compute world acceleration and remove gravity\n            acc_world = compute_world_acceleration(acc,rot)\n            acc_linear = remove_gravity_from_acc(acc,rot)\n\n            # Add angular velocity and distance\n            angular_vel = calculate_angular_velocity_from_quat(rot)\n            angular_dist = calculate_angular_distance(rot)\n\n            # Concatenate features: acc_world + linear acc + angular vel + distance\n            acc_features = np.concatenate([acc_world, acc_linear, angular_vel, angular_dist[:,None]], axis=1)\n\n            # Padding\n            acc_pad = np.zeros((self.window_len, acc_features.shape[1]),dtype=np.float32)\n            rot_pad = np.zeros((self.window_len, 4),dtype=np.float32)\n            mask = np.zeros(self.window_len,dtype=np.bool_)\n\n            insert = start_c-start\n            acc_pad[insert:insert+len(acc_features)] = acc_features\n            rot_pad[insert:insert+len(rot)] = rot\n            mask[insert:insert+len(acc_features)] = True\n\n            label = seq[\"gesture_id\"].mode().iloc[0]\n            self.windows.append((acc_pad,rot_pad,mask,label))\n\n        print(f\"Created {len(self.windows)} multimodal windows\")\n\n    def __len__(self): return len(self.windows)\n    def __getitem__(self,idx):\n        acc, rot, mask, label = self.windows[idx]\n        return {\n            \"acc\": torch.tensor(acc,dtype=torch.float32),\n            \"rot\": torch.tensor(rot,dtype=torch.float32),\n            \"mask\": torch.tensor(mask,dtype=torch.bool),\n            \"label\": torch.tensor(label,dtype=torch.long)\n        }\n\n# ============================================================\n# MODEL\n# ============================================================\n\nclass AxisCNN(nn.Module):\n    def __init__(self, hidden_dim=32):\n        super().__init__()\n        self.hidden_dim = hidden_dim\n        \n        self.net = nn.Sequential(\n            nn.Conv1d(1, hidden_dim, kernel_size=5, padding=2),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Conv1d(hidden_dim, hidden_dim, kernel_size=5, padding=2),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Conv1d(hidden_dim, hidden_dim, kernel_size=3, padding=1),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n        )\n\n    def forward(self, x):\n        \"\"\"\n        x: (B, 1, T)\n        return: (B, hidden_dim)\n        \"\"\"\n        x = self.net(x)\n        x = x.mean(dim=-1)  # temporal pooling\n        return x\n\n\nclass EnhancedCNNEncoder(nn.Module):\n    def __init__(self, input_dim, d_model=64, axis_hidden_dim=32):\n        super().__init__()\n        self.input_dim = input_dim\n        self.d_model = d_model\n        \n        # AxisCNN for per-axis feature extraction\n        self.axis_cnns = nn.ModuleList([\n            AxisCNN(hidden_dim=axis_hidden_dim) for _ in range(input_dim)\n        ])\n        \n        # Dimension adjustment layer\n        self.axis_proj = nn.Linear(input_dim * axis_hidden_dim, d_model)\n        \n        # Temporal CNN for sequence modeling\n        self.temporal_cnn = nn.Sequential(\n            nn.Conv1d(d_model, d_model, kernel_size=3, padding=1),\n            nn.BatchNorm1d(d_model),\n            nn.ReLU(),\n            nn.Conv1d(d_model, d_model, kernel_size=3, padding=1),\n            nn.BatchNorm1d(d_model),\n            nn.ReLU(),\n        )\n        \n        self.ln = nn.LayerNorm(d_model)\n\n    def forward(self, x):\n        \"\"\"\n        x: (B, T, F)\n        return: (B, T, d_model)\n        \"\"\"\n        B, T, F = x.shape\n        \n        # Extract per-axis features\n        axis_features = []\n        for i in range(F):\n            # Extract axis i: (B, T, 1) -> (B, 1, T)\n            axis_data = x[:, :, i:i+1].transpose(1, 2)\n            # Process through axis-specific CNN\n            axis_feat = self.axis_cnns[i](axis_data)  # (B, axis_hidden_dim)\n            axis_features.append(axis_feat)\n        \n        # Stack axis features: (B, F * axis_hidden_dim)\n        axis_features = torch.cat(axis_features, dim=1)\n        \n        # Project to d_model dimension\n        projected = self.axis_proj(axis_features)  # (B, d_model)\n        \n        # Repeat across temporal dimension\n        expanded = projected.unsqueeze(1).repeat(1, T, 1)  # (B, T, d_model)\n        \n        # Apply temporal CNN\n        expanded = expanded.transpose(1, 2)  # (B, d_model, T)\n        expanded = self.temporal_cnn(expanded)\n        expanded = expanded.transpose(1, 2)  # (B, T, d_model)\n        \n        return self.ln(expanded)\n\n\nclass TransformerEncoder(nn.Module):\n    def __init__(self, d_model=64, nhead=4, layers=2):\n        super().__init__()\n        enc_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            batch_first=True,\n            dim_feedforward=d_model*4,\n            activation=\"gelu\"\n        )\n        self.encoder = nn.TransformerEncoder(enc_layer, layers)\n        \n    def forward(self, x, mask):\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        m = mask.unsqueeze(-1).float()\n        return (x * m).sum(dim=1) / m.sum(dim=1).clamp(min=1)\n\n\nclass HybridFusedModel(nn.Module):\n    def __init__(self, num_classes, acc_feat_dim, rot_dim=4, d_model=64, axis_hidden_dim=32):\n        super().__init__()\n        \n        # Enhanced encoders with AxisCNN integration\n        self.acc_encoder = EnhancedCNNEncoder(acc_feat_dim, d_model, axis_hidden_dim)\n        self.rot_encoder = EnhancedCNNEncoder(rot_dim, d_model, axis_hidden_dim)\n        \n        # Transformer encoders\n        self.acc_trans = TransformerEncoder(d_model)\n        self.rot_trans = TransformerEncoder(d_model)\n        \n        # Fusion and classification\n        self.fusion = nn.Sequential(\n            nn.LayerNorm(d_model * 2),\n            nn.Linear(d_model * 2, d_model),\n            nn.GELU(),\n            nn.Dropout(0.2)\n        )\n        \n        self.head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model // 2),\n            nn.GELU(),\n            nn.Dropout(0.1),\n            nn.Linear(d_model // 2, num_classes)\n        )\n\n    def forward(self, acc, rot, mask):\n        # Process through enhanced CNN encoders\n        acc_feat = self.acc_encoder(acc)\n        rot_feat = self.rot_encoder(rot)\n        \n        # Process through transformers\n        acc_seq = self.acc_trans(acc_feat, mask)\n        rot_seq = self.rot_trans(rot_feat, mask)\n        \n        # Fuse features\n        z = torch.cat([acc_seq, rot_seq], dim=1)\n        z = self.fusion(z)\n        \n        return self.head(z)\n\n# ============================================================\n# TRAINING\n# ============================================================\n\ndataset = AccRotHybridDataset(CSV_PATH,WINDOW_BEFORE,WINDOW_AFTER)\nnum_classes = len(dataset.gesture_classes)\nacc_feat_dim = dataset[0][\"acc\"].shape[1]\n\ntrain_size = int(0.8*len(dataset)); test_size=len(dataset)-train_size\ntrain_ds, test_ds = random_split(dataset,[train_size,test_size])\ntrain_loader = DataLoader(train_ds,batch_size=BATCH_SIZE,shuffle=True)\ntest_loader = DataLoader(test_ds,batch_size=BATCH_SIZE)\n\nmodel = HybridFusedModel(num_classes,acc_feat_dim).to(DEVICE)\noptimizer = torch.optim.AdamW(model.parameters(),lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train(); total_loss=0\n    for batch in train_loader:\n        acc=batch[\"acc\"].to(DEVICE); rot=batch[\"rot\"].to(DEVICE)\n        mask=batch[\"mask\"].to(DEVICE); label=batch[\"label\"].to(DEVICE)\n\n        optimizer.zero_grad()\n        logits = model(acc,rot,mask)\n        loss = criterion(logits,label)\n        loss.backward()\n        optimizer.step()\n        total_loss+=loss.item()\n    print(f\"Epoch {epoch+1} | Loss: {total_loss/len(train_loader):.4f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\nmodel.eval()\nall_preds, all_labels=[],[]\nwith torch.no_grad():\n    for batch in test_loader:\n        acc=batch[\"acc\"].to(DEVICE); rot=batch[\"rot\"].to(DEVICE)\n        mask=batch[\"mask\"].to(DEVICE)\n        logits = model(acc,rot,mask)\n        preds = logits.argmax(dim=1)\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(batch[\"label\"].numpy())\n\nall_preds=np.array(all_preds); all_labels=np.array(all_labels)\noverall_acc = (all_preds==all_labels).mean()\nprint(f\"\\nOverall accuracy: {overall_acc:.3f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.656629Z","iopub.status.idle":"2026-02-02T17:51:47.656855Z","shell.execute_reply.started":"2026-02-02T17:51:47.656748Z","shell.execute_reply":"2026-02-02T17:51:47.656758Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FLEXIBLE HYBRID CNN + TRANSFORMER MODEL FOR KAGGLE\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom scipy.spatial.transform import Rotation as R\nimport time\n\n# ============================================================\n# CONFIGURABLE VARIABLES\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\nWINDOW_BEFORE = 10\nWINDOW_AFTER = 10\nBATCH_SIZE = 32\nEPOCHS = 20\nLR = 3e-4\nD_MODEL = 64\nAXIS_HIDDEN_DIM = 32\nUSE_AXISCNN = True\nUSE_TRANSFORMER = True\nFUSION = 'concat'  # 'concat' or 'gated'\nDROPOUT = 0.1\nUSE_ACC = True\nUSE_ROT = True\nVAL_RATIO = 0.2\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nSEED = 42\n\n# ============================================================\n# HELPER FUNCTIONS\n# ============================================================\n\ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    rot_cleaned = rot_data.copy()\n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        if missing_count == 0:\n            norm = np.linalg.norm(row)\n            rot_cleaned[i] = row / norm if norm > 1e-8 else [1.0,0.0,0.0,0.0]\n        elif missing_count == 1:\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            sum_squares = np.sum(valid_values**2)\n            if sum_squares <= 1.0:\n                missing_value = np.sqrt(max(0,1.0-sum_squares))\n                if i>0 and not np.isnan(rot_cleaned[i-1,missing_idx]):\n                    if rot_cleaned[i-1,missing_idx]<0: missing_value=-missing_value\n                rot_cleaned[i,missing_idx]=missing_value\n                rot_cleaned[i,~np.isnan(row)] = valid_values\n            else: rot_cleaned[i]=[1.0,0.0,0.0,0.0]\n        else:\n            rot_cleaned[i]=[1.0,0.0,0.0,0.0]\n    return rot_cleaned\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n    try:\n        rot_scipy = rot[:,[1,2,3,0]]\n        norms = np.linalg.norm(rot_scipy, axis=1)\n        mask = norms<1e-8\n        rot_scipy[mask] = [0.0,0.0,0.0,1.0]\n        r = R.from_quat(rot_scipy)\n        return r.apply(acc)\n    except:\n        return acc.copy()\n\ndef remove_gravity_from_acc(acc_data, rot_data):\n    num_samples = acc_data.shape[0]\n    linear_accel = np.zeros_like(acc_data)\n    gravity_world = np.array([0,0,9.81])\n    for i in range(num_samples):\n        if np.all(np.isnan(rot_data[i])) or np.all(np.isclose(rot_data[i],0)):\n            linear_accel[i,:] = acc_data[i,:]\n            continue\n        try:\n            rotation = R.from_quat(rot_data[i])\n            gravity_sensor_frame = rotation.apply(gravity_world,inverse=True)\n            linear_accel[i,:] = acc_data[i,:]-gravity_sensor_frame\n        except:\n            linear_accel[i,:] = acc_data[i,:]\n    return linear_accel\n\ndef calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n    num_samples = rot_data.shape[0]\n    angular_vel = np.zeros((num_samples,3))\n    for i in range(num_samples-1):\n        q_t = rot_data[i]; q_t_plus = rot_data[i+1]\n        if np.any(np.isnan(q_t)) or np.any(np.isnan(q_t_plus)): continue\n        try:\n            r1 = R.from_quat(q_t); r2 = R.from_quat(q_t_plus)\n            delta_rot = r1.inv()*r2\n            angular_vel[i,:] = delta_rot.as_rotvec()/time_delta\n        except: continue\n    return angular_vel\n\ndef calculate_angular_distance(rot_data):\n    num_samples = rot_data.shape[0]\n    angular_dist = np.zeros(num_samples)\n    for i in range(num_samples-1):\n        q1=rot_data[i]; q2=rot_data[i+1]\n        if np.any(np.isnan(q1)) or np.any(np.isnan(q2)): continue\n        try:\n            r1=R.from_quat(q1); r2=R.from_quat(q2)\n            rel_rot = r1.inv()*r2\n            angular_dist[i]=np.linalg.norm(rel_rot.as_rotvec())\n        except: continue\n    return angular_dist\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass AccRotHybridDataset(Dataset):\n    ACC_COLS = [\"acc_x\",\"acc_y\",\"acc_z\"]\n    ROT_COLS = [\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]\n\n    def __init__(self,csv_path,window_before=10,window_after=10):\n        self.df=pd.read_csv(csv_path)\n        self.window_before = window_before\n        self.window_after = window_after\n        self.window_len = window_before+window_after+1\n\n        # Encode gestures\n        self.df[\"gesture\"] = self.df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        self.df[\"gesture_id\"] = self.le.fit_transform(self.df[\"gesture\"])\n        self.gesture_classes = self.le.classes_\n\n        self.sequences = list(self.df.groupby(\"sequence_id\"))\n        self.windows=[]\n\n        for _,seq in self.sequences:\n            seq = seq.reset_index(drop=True)\n            gesture_rows = seq[seq[\"phase\"]==\"Gesture\"]\n            if gesture_rows.empty: continue\n            center = gesture_rows.index[0]\n            start=center-self.window_before; end=center+self.window_after+1\n            start_c = max(0,start); end_c=min(len(seq),end)\n\n            acc = seq.iloc[start_c:end_c][self.ACC_COLS].values.astype(np.float32)\n            rot = seq.iloc[start_c:end_c][self.ROT_COLS].values.astype(np.float32)\n\n            rot = handle_quaternion_missing_values(rot)\n            norms = np.linalg.norm(rot,axis=1,keepdims=True); norms[norms<1e-6]=1.0\n            rot = rot/norms\n\n            acc_world = compute_world_acceleration(acc,rot)\n            acc_linear = remove_gravity_from_acc(acc,rot)\n            angular_vel = calculate_angular_velocity_from_quat(rot)\n            angular_dist = calculate_angular_distance(rot)\n\n            acc_features = np.concatenate([acc_world, acc_linear, angular_vel, angular_dist[:,None]], axis=1)\n\n            acc_pad = np.zeros((self.window_len, acc_features.shape[1]),dtype=np.float32)\n            rot_pad = np.zeros((self.window_len, 4),dtype=np.float32)\n            mask = np.zeros(self.window_len,dtype=np.bool_)\n\n            insert = start_c-start\n            acc_pad[insert:insert+len(acc_features)] = acc_features\n            rot_pad[insert:insert+len(rot)] = rot\n            mask[insert:insert+len(acc_features)] = True\n\n            label = seq[\"gesture_id\"].mode().iloc[0]\n            self.windows.append((acc_pad,rot_pad,mask,label))\n\n        print(f\"Created {len(self.windows)} multimodal windows\")\n\n    def __len__(self): return len(self.windows)\n    def __getitem__(self, idx):\n        acc, rot, mask, label = self.windows[idx]\n    \n        if not USE_ACC:\n            acc = np.zeros_like(acc, dtype=np.float32)\n    \n        if not USE_ROT:\n            rot = np.zeros_like(rot, dtype=np.float32)\n    \n        return {\n            \"acc\": torch.tensor(acc, dtype=torch.float32),\n            \"rot\": torch.tensor(rot, dtype=torch.float32),\n            \"mask\": torch.tensor(mask, dtype=torch.bool),\n            \"label\": torch.tensor(label, dtype=torch.long),\n        }\n\n# ============================================================\n# MODEL\n# ============================================================\n\nclass AxisCNN(nn.Module):\n    def __init__(self, hidden_dim=32):\n        super().__init__()\n        self.hidden_dim = hidden_dim\n        self.net = nn.Sequential(\n            nn.Conv1d(1, hidden_dim, kernel_size=5, padding=2),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Conv1d(hidden_dim, hidden_dim, kernel_size=5, padding=2),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Conv1d(hidden_dim, hidden_dim, kernel_size=3, padding=1),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n        )\n    def forward(self,x):\n        x = self.net(x)\n        x = x.mean(dim=-1)\n        return x\n\nclass EnhancedCNNEncoder(nn.Module):\n    def __init__(self, input_dim, d_model=64, axis_hidden_dim=32):\n        super().__init__()\n        self.axis_cnns = nn.ModuleList([AxisCNN(hidden_dim=axis_hidden_dim) for _ in range(input_dim)])\n        self.axis_proj = nn.Linear(input_dim*axis_hidden_dim, d_model)\n        self.temporal_cnn = nn.Sequential(\n            nn.Conv1d(d_model,d_model,3,padding=1), nn.BatchNorm1d(d_model), nn.ReLU(),\n            nn.Conv1d(d_model,d_model,3,padding=1), nn.BatchNorm1d(d_model), nn.ReLU()\n        )\n        self.ln = nn.LayerNorm(d_model)\n    def forward(self,x):\n        B,T,F = x.shape\n        feats = []\n        for i in range(F):\n            axis_data = x[:,:,i:i+1].transpose(1,2)\n            feats.append(self.axis_cnns[i](axis_data))\n        projected = self.axis_proj(torch.cat(feats,dim=1))\n        expanded = projected.unsqueeze(1).repeat(1,T,1)\n        expanded = self.temporal_cnn(expanded.transpose(1,2)).transpose(1,2)\n        return self.ln(expanded)\n\nclass TransformerEncoder(nn.Module):\n    def __init__(self,d_model=64,nhead=4,layers=2):\n        super().__init__()\n        enc_layer = nn.TransformerEncoderLayer(d_model=d_model,nhead=nhead,batch_first=True,dim_feedforward=d_model*4,activation='gelu')\n        self.encoder = nn.TransformerEncoder(enc_layer,layers)\n    def forward(self,x,mask):\n        x = self.encoder(x, src_key_padding_mask=~mask)\n        m = mask.unsqueeze(-1).float()\n        return (x*m).sum(dim=1)/m.sum(dim=1).clamp(min=1)\n\nclass HybridFusedModel(nn.Module):\n    def __init__(self,num_classes,acc_feat_dim,rot_dim=4,d_model=64):\n        super().__init__()\n        self.use_acc = USE_ACC\n        self.use_rot = USE_ROT\n        self.acc_encoder = EnhancedCNNEncoder(acc_feat_dim,d_model) if USE_ACC else None\n        self.rot_encoder = EnhancedCNNEncoder(rot_dim,d_model) if USE_ROT else None\n        self.acc_trans = TransformerEncoder(d_model) if USE_TRANSFORMER and USE_ACC else None\n        self.rot_trans = TransformerEncoder(d_model) if USE_TRANSFORMER and USE_ROT else None\n        self.fusion = nn.Sequential(nn.LayerNorm(d_model*2), nn.Linear(d_model*2,d_model), nn.GELU(), nn.Dropout(DROPOUT))\n        self.head = nn.Sequential(nn.LayerNorm(d_model), nn.Linear(d_model,d_model//2), nn.GELU(), nn.Dropout(DROPOUT), nn.Linear(d_model//2,num_classes))\n    def forward(self, acc, rot, mask):\n        feats=[]\n        if self.use_acc:\n            acc_feat = self.acc_encoder(acc)\n            acc_out = self.acc_trans(acc_feat,mask) if self.acc_trans else (acc_feat*mask.unsqueeze(-1).float()).sum(1)/mask.sum(1,keepdim=True).clamp(min=1)\n            feats.append(acc_out)\n        if self.use_rot:\n            rot_feat = self.rot_encoder(rot)\n            rot_out = self.rot_trans(rot_feat,mask) if self.rot_trans else (rot_feat*mask.unsqueeze(-1).float()).sum(1)/mask.sum(1,keepdim=True).clamp(min=1)\n            feats.append(rot_out)\n        if len(feats)==2:\n            z = torch.cat(feats,dim=1)\n            z = self.fusion(z)\n        else:\n            z = feats[0]\n        return self.head(z)\n\n# ============================================================\n# TRAINING\n# ============================================================\nWINDOW_BEFORE = 20\nWINDOW_AFTER = 20\nEPOCHS = 20\nUSE_AXISCNN = False\nUSE_TRANSFORMER = True\nFUSION = 'concat'  # 'concat' or 'gated'\nUSE_ACC = True\nUSE_ROT = False\n\ntorch.manual_seed(SEED)\ndataset = AccRotHybridDataset(CSV_PATH,WINDOW_BEFORE,WINDOW_AFTER)\nnum_classes = len(dataset.gesture_classes)\nacc_feat_dim = dataset[0][\"acc\"].shape[1] if USE_ACC else 0\n\ntrain_size = int((1-VAL_RATIO)*len(dataset)); test_size=len(dataset)-train_size\ntrain_ds, test_ds = random_split(dataset,[train_size,test_size])\ntrain_loader = DataLoader(train_ds,batch_size=BATCH_SIZE,shuffle=True)\ntest_loader = DataLoader(test_ds,batch_size=BATCH_SIZE)\n\nmodel = HybridFusedModel(num_classes,acc_feat_dim).to(DEVICE)\noptimizer = torch.optim.AdamW(model.parameters(),lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train(); total_loss=0\n    for batch in train_loader:\n        acc=batch[\"acc\"].to(DEVICE) if USE_ACC else None\n        rot=batch[\"rot\"].to(DEVICE) if USE_ROT else None\n        mask=batch[\"mask\"].to(DEVICE); label=batch[\"label\"].to(DEVICE)\n        optimizer.zero_grad()\n        logits = model(acc,rot,mask)\n        loss = criterion(logits,label)\n        loss.backward(); optimizer.step()\n        total_loss+=loss.item()\n    print(f\"Epoch {epoch+1} | Loss: {total_loss/len(train_loader):.4f}\")\n\n# ============================================================\n# EVALUATION\n# ============================================================\n\n\nmodel.eval()\nall_preds, all_labels=[],[]\nwith torch.no_grad():\n    for batch in test_loader:\n        acc=batch[\"acc\"].to(DEVICE) if USE_ACC else None\n        rot=batch[\"rot\"].to(DEVICE) if USE_ROT else None\n        mask=batch[\"mask\"].to(DEVICE)\n        logits = model(acc,rot,mask)\n        preds = logits.argmax(dim=1)\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(batch[\"label\"].numpy())\n\nall_preds=np.array(all_preds); all_labels=np.array(all_labels)\noverall_acc = (all_preds==all_labels).mean()\nprint(f\"\\nOverall accuracy: {overall_acc:.3f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.657844Z","iopub.status.idle":"2026-02-02T17:51:47.658168Z","shell.execute_reply.started":"2026-02-02T17:51:47.658011Z","shell.execute_reply":"2026-02-02T17:51:47.658025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# HYBRID ACC + ROT + ToF MODEL (KAGGLE, SINGLE FILE)\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom scipy.spatial.transform import Rotation as R\nimport random\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\n\nWINDOW_BEFORE = 20\nWINDOW_AFTER = 20\nBATCH_SIZE = 32\nEPOCHS = 20\nLR = 3e-4\n\nD_MODEL = 64\nAXIS_HIDDEN_DIM = 32\nDROPOUT = 0.1\n\nUSE_ACC = True\nUSE_ROT = False\nUSE_TOF = True          # ⭐ NEW\nFREEZE_TOF = False     # freeze autoencoder encoder\n\nVAL_RATIO = 0.2\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nSEED = 42\n\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\nrandom.seed(SEED)\n\n# ============================================================\n# ToF AUTOENCODER\n# ============================================================\n\nclass ToFAutoencoder(nn.Module):\n    def __init__(self, input_dim, latent_dim=8, hidden_dim=128):\n        super().__init__()\n\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, latent_dim),\n        )\n\n        self.decoder = nn.Sequential(\n            nn.Linear(latent_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, input_dim),\n        )\n\n    def forward(self, x):\n        z = self.encoder(x)\n        recon = self.decoder(z)\n        return recon, z\n\n# ============================================================\n# HELPERS\n# ============================================================\n\ndef handle_quaternion_missing_values(rot):\n    out = rot.copy()\n    for i in range(len(rot)):\n        r = rot[i]\n        if np.isnan(r).sum() == 0:\n            n = np.linalg.norm(r)\n            out[i] = r / n if n > 1e-6 else [1,0,0,0]\n        else:\n            out[i] = [1,0,0,0]\n    return out\n\ndef compute_world_acc(acc, rot):\n    try:\n        r = R.from_quat(rot[:, [1,2,3,0]])\n        return r.apply(acc)\n    except:\n        return acc.copy()\n\ndef remove_gravity(acc, rot):\n    g = np.array([0,0,9.81])\n    out = np.zeros_like(acc)\n    for i in range(len(acc)):\n        try:\n            r = R.from_quat(rot[i])\n            out[i] = acc[i] - r.apply(g, inverse=True)\n        except:\n            out[i] = acc[i]\n    return out\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass HybridDataset(Dataset):\n\n    ACC_COLS = [\"acc_x\",\"acc_y\",\"acc_z\"]\n    ROT_COLS = [\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]\n\n    def __init__(self, csv_path):\n        df = pd.read_csv(csv_path)\n\n        # labels\n        df[\"gesture\"] = df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        df[\"label\"] = self.le.fit_transform(df[\"gesture\"])\n        self.classes = self.le.classes_\n\n        # ToF\n        self.tof_cols = [c for c in df.columns if c.startswith(\"tof\")]\n        self.use_tof = len(self.tof_cols) > 0\n\n        if self.use_tof:\n            scaler = StandardScaler()\n            df[self.tof_cols] = scaler.fit_transform(\n                df[self.tof_cols].fillna(0.0)\n            )\n\n        self.windows = []\n        win_len = WINDOW_BEFORE + WINDOW_AFTER + 1\n\n        for _, seq in df.groupby(\"sequence_id\"):\n            seq = seq.reset_index(drop=True)\n            if \"Gesture\" not in seq[\"phase\"].values:\n                continue\n\n            center = seq[seq[\"phase\"]==\"Gesture\"].index[0]\n            s = max(0, center - WINDOW_BEFORE)\n            e = min(len(seq), center + WINDOW_AFTER + 1)\n\n            # ACC\n            acc = seq.iloc[s:e][self.ACC_COLS].values.astype(np.float32)\n            rot = seq.iloc[s:e][self.ROT_COLS].values.astype(np.float32)\n\n            rot = handle_quaternion_missing_values(rot)\n            acc_w = compute_world_acc(acc, rot)\n            acc_l = remove_gravity(acc, rot)\n\n            acc_feat = np.concatenate([acc_w, acc_l], axis=1)\n\n            acc_pad = np.zeros((win_len, acc_feat.shape[1]), np.float32)\n            rot_pad = np.zeros((win_len, 4), np.float32)\n            mask = np.zeros(win_len, bool)\n\n            insert = WINDOW_BEFORE - (center - s)\n            acc_pad[insert:insert+len(acc_feat)] = acc_feat\n            rot_pad[insert:insert+len(rot)] = rot\n            mask[insert:insert+len(acc_feat)] = True\n\n            # ToF (center frame only)\n            if self.use_tof:\n                tof = seq.iloc[center][self.tof_cols].values.astype(np.float32)\n            else:\n                tof = np.zeros(1, dtype=np.float32)\n\n            label = seq[\"label\"].mode().iloc[0]\n            self.windows.append((acc_pad, rot_pad, tof, mask, label))\n\n        print(f\"Created {len(self.windows)} samples\")\n\n    def __len__(self):\n        return len(self.windows)\n\n    def __getitem__(self, idx):\n        acc, rot, tof, mask, label = self.windows[idx]\n\n        return {\n            \"acc\": torch.tensor(acc),\n            \"rot\": torch.tensor(rot),\n            \"tof\": torch.tensor(tof),\n            \"mask\": torch.tensor(mask),\n            \"label\": torch.tensor(label)\n        }\n\n# ============================================================\n# ENCODERS\n# ============================================================\n\nclass AxisCNN(nn.Module):\n    def __init__(self, hidden=32):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Conv1d(1, hidden, 5, padding=2),\n            nn.ReLU(),\n            nn.Conv1d(hidden, hidden, 5, padding=2),\n            nn.ReLU()\n        )\n    def forward(self, x):\n        return self.net(x).mean(-1)\n\nclass CNNEncoder(nn.Module):\n    def __init__(self, feat_dim, d_model):\n        super().__init__()\n        self.cnns = nn.ModuleList([AxisCNN() for _ in range(feat_dim)])\n        self.proj = nn.Linear(feat_dim * 32, d_model)\n\n    def forward(self, x):\n        feats = []\n        for i in range(x.shape[2]):\n            feats.append(self.cnns[i](x[:,:,i:i+1].transpose(1,2)))\n        return self.proj(torch.cat(feats, dim=1))\n\n# ============================================================\n# FULL MODEL\n# ============================================================\n\nclass HybridModel(nn.Module):\n    def __init__(self, num_classes, acc_dim, tof_dim):\n        super().__init__()\n\n        self.acc_enc = CNNEncoder(acc_dim, D_MODEL) if USE_ACC else None\n        self.rot_enc = CNNEncoder(4, D_MODEL) if USE_ROT else None\n\n        self.tof_ae = ToFAutoencoder(tof_dim) if USE_TOF else None\n        self.tof_proj = nn.Linear(8, D_MODEL) if USE_TOF else None\n\n        fusion_dim = D_MODEL * sum([USE_ACC, USE_ROT, USE_TOF])\n        self.head = nn.Sequential(\n            nn.LayerNorm(fusion_dim),\n            nn.Linear(fusion_dim, D_MODEL),\n            nn.GELU(),\n            nn.Dropout(DROPOUT),\n            nn.Linear(D_MODEL, num_classes)\n        )\n\n    def forward(self, acc, rot, tof):\n        feats = []\n\n        if USE_ACC:\n            feats.append(self.acc_enc(acc))\n        if USE_ROT:\n            feats.append(self.rot_enc(rot))\n        if USE_TOF:\n            _, z = self.tof_ae(tof)\n            feats.append(self.tof_proj(z))\n\n        z = torch.cat(feats, dim=1)\n        return self.head(z)\n\n# ============================================================\n# TRAIN\n# ============================================================\n\ndataset = HybridDataset(CSV_PATH)\nnum_classes = len(dataset.classes)\nacc_dim = dataset[0][\"acc\"].shape[1]\ntof_dim = dataset[0][\"tof\"].shape[0]\n\ntrain_size = int((1 - VAL_RATIO) * len(dataset))\ntrain_ds, val_ds = random_split(dataset, [train_size, len(dataset)-train_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\nval_loader = DataLoader(val_ds, batch_size=BATCH_SIZE)\n\nmodel = HybridModel(num_classes, acc_dim, tof_dim).to(DEVICE)\n\nif USE_TOF and FREEZE_TOF:\n    for p in model.tof_ae.encoder.parameters():\n        p.requires_grad = False\n\nopt = torch.optim.AdamW(model.parameters(), lr=LR)\nclf_loss = nn.CrossEntropyLoss()\ntof_loss = nn.MSELoss()\n\n# ============================================================\n# LOOP\n# ============================================================\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total = 0\n\n    for b in train_loader:\n        acc = b[\"acc\"].to(DEVICE)\n        rot = b[\"rot\"].to(DEVICE)\n        tof = b[\"tof\"].to(DEVICE)\n        y = b[\"label\"].to(DEVICE)\n\n        opt.zero_grad()\n        logits = model(acc, rot, tof)\n        loss = clf_loss(logits, y)\n\n        if USE_TOF:\n            recon, _ = model.tof_ae(tof)\n            loss = loss + 0.1 * tof_loss(recon, tof)\n\n        loss.backward()\n        opt.step()\n        total += loss.item()\n\n    print(f\"Epoch {epoch+1} | Loss {total/len(train_loader):.4f}\")\n\n# ============================================================\n# EVAL\n# ============================================================\n\nmodel.eval()\ncorrect, total = 0, 0\nwith torch.no_grad():\n    for b in val_loader:\n        logits = model(\n            b[\"acc\"].to(DEVICE),\n            b[\"rot\"].to(DEVICE),\n            b[\"tof\"].to(DEVICE)\n        )\n        pred = logits.argmax(1)\n        correct += (pred.cpu() == b[\"label\"]).sum().item()\n        total += len(pred)\n\nprint(f\"\\nValidation accuracy: {correct/total:.3f}\")\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-02-02T17:51:47.660013Z","iopub.status.idle":"2026-02-02T17:51:47.660329Z","shell.execute_reply.started":"2026-02-02T17:51:47.660156Z","shell.execute_reply":"2026-02-02T17:51:47.660171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# HYBRID ACC + ROT + ToF MODEL (KAGGLE, SINGLE FILE)\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom scipy.spatial.transform import Rotation as R\nimport random\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nCSV_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\n\nWINDOW_BEFORE = 20\nWINDOW_AFTER = 20\nBATCH_SIZE = 32\nEPOCHS = 20\nLR = 3e-4\n\nD_MODEL = 64\nAXIS_HIDDEN_DIM = 32\nDROPOUT = 0.1\n\nUSE_ACC = True\nUSE_ROT = False\nUSE_TOF = True          # ⭐ NEW\nFREEZE_TOF = False     # freeze autoencoder encoder\n\nVAL_RATIO = 0.2\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nSEED = 42\n\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\nrandom.seed(SEED)\n\n# ============================================================\n# ToF AUTOENCODER\n# ============================================================\n\nclass ToFAutoencoder(nn.Module):\n    def __init__(self, input_dim, latent_dim=8, hidden_dim=128):\n        super().__init__()\n\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, latent_dim),\n        )\n\n        self.decoder = nn.Sequential(\n            nn.Linear(latent_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, input_dim),\n        )\n\n    def forward(self, x):\n        z = self.encoder(x)\n        recon = self.decoder(z)\n        return recon, z\n\n# ============================================================\n# HELPERS\n# ============================================================\n\ndef handle_quaternion_missing_values(rot):\n    out = rot.copy()\n    for i in range(len(rot)):\n        r = rot[i]\n        if np.isnan(r).sum() == 0:\n            n = np.linalg.norm(r)\n            out[i] = r / n if n > 1e-6 else [1,0,0,0]\n        else:\n            out[i] = [1,0,0,0]\n    return out\n\ndef compute_world_acc(acc, rot):\n    try:\n        r = R.from_quat(rot[:, [1,2,3,0]])\n        return r.apply(acc)\n    except:\n        return acc.copy()\n\ndef remove_gravity(acc, rot):\n    g = np.array([0,0,9.81])\n    out = np.zeros_like(acc)\n    for i in range(len(acc)):\n        try:\n            r = R.from_quat(rot[i])\n            out[i] = acc[i] - r.apply(g, inverse=True)\n        except:\n            out[i] = acc[i]\n    return out\n\n# ============================================================\n# DATASET\n# ============================================================\n\nclass HybridDataset(Dataset):\n\n    ACC_COLS = [\"acc_x\",\"acc_y\",\"acc_z\"]\n    ROT_COLS = [\"rot_w\",\"rot_x\",\"rot_y\",\"rot_z\"]\n\n    def __init__(self, csv_path):\n        df = pd.read_csv(csv_path)\n\n        # labels\n        df[\"gesture\"] = df[\"gesture\"].astype(str)\n        self.le = LabelEncoder()\n        df[\"label\"] = self.le.fit_transform(df[\"gesture\"])\n        self.classes = self.le.classes_\n\n        # ToF\n        self.tof_cols = [c for c in df.columns if c.startswith(\"tof\")]\n        self.use_tof = len(self.tof_cols) > 0\n\n        if self.use_tof:\n            scaler = StandardScaler()\n            df[self.tof_cols] = scaler.fit_transform(\n                df[self.tof_cols].fillna(0.0)\n            )\n\n        self.windows = []\n        win_len = WINDOW_BEFORE + WINDOW_AFTER + 1\n\n        for _, seq in df.groupby(\"sequence_id\"):\n            seq = seq.reset_index(drop=True)\n            if \"Gesture\" not in seq[\"phase\"].values:\n                continue\n\n            center = seq[seq[\"phase\"]==\"Gesture\"].index[0]\n            s = max(0, center - WINDOW_BEFORE)\n            e = min(len(seq), center + WINDOW_AFTER + 1)\n\n            # ACC\n            acc = seq.iloc[s:e][self.ACC_COLS].values.astype(np.float32)\n            rot = seq.iloc[s:e][self.ROT_COLS].values.astype(np.float32)\n\n            rot = handle_quaternion_missing_values(rot)\n            acc_w = compute_world_acc(acc, rot)\n            acc_l = remove_gravity(acc, rot)\n\n            acc_feat = np.concatenate([acc_w, acc_l], axis=1)\n\n            acc_pad = np.zeros((win_len, acc_feat.shape[1]), np.float32)\n            rot_pad = np.zeros((win_len, 4), np.float32)\n            mask = np.zeros(win_len, bool)\n\n            insert = WINDOW_BEFORE - (center - s)\n            acc_pad[insert:insert+len(acc_feat)] = acc_feat\n            rot_pad[insert:insert+len(rot)] = rot\n            mask[insert:insert+len(acc_feat)] = True\n\n            # ToF (center frame only)\n            if self.use_tof:\n                tof = seq.iloc[center][self.tof_cols].values.astype(np.float32)\n            else:\n                tof = np.zeros(1, dtype=np.float32)\n\n            label = seq[\"label\"].mode().iloc[0]\n            self.windows.append((acc_pad, rot_pad, tof, mask, label))\n\n        print(f\"Created {len(self.windows)} samples\")\n\n    def __len__(self):\n        return len(self.windows)\n\n    def __getitem__(self, idx):\n        acc, rot, tof, mask, label = self.windows[idx]\n\n        return {\n            \"acc\": torch.tensor(acc),\n            \"rot\": torch.tensor(rot),\n            \"tof\": torch.tensor(tof),\n            \"mask\": torch.tensor(mask),\n            \"label\": torch.tensor(label)\n        }\n\n# ============================================================\n# ENCODERS\n# ============================================================\n\nclass AxisCNN(nn.Module):\n    def __init__(self, hidden=32):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Conv1d(1, hidden, 5, padding=2),\n            nn.ReLU(),\n            nn.Conv1d(hidden, hidden, 5, padding=2),\n            nn.ReLU()\n        )\n    def forward(self, x):\n        return self.net(x).mean(-1)\n\nclass CNNEncoder(nn.Module):\n    def __init__(self, feat_dim, d_model):\n        super().__init__()\n        self.cnns = nn.ModuleList([AxisCNN() for _ in range(feat_dim)])\n        self.proj = nn.Linear(feat_dim * 32, d_model)\n\n    def forward(self, x):\n        feats = []\n        for i in range(x.shape[2]):\n            feats.append(self.cnns[i](x[:,:,i:i+1].transpose(1,2)))\n        return self.proj(torch.cat(feats, dim=1))\n\n# ============================================================\n# FULL MODEL\n# ============================================================\n\nclass HybridModel(nn.Module):\n    def __init__(self, num_classes, acc_dim, tof_dim):\n        super().__init__()\n\n        self.acc_enc = CNNEncoder(acc_dim, D_MODEL) if USE_ACC else None\n        self.rot_enc = CNNEncoder(4, D_MODEL) if USE_ROT else None\n\n        self.tof_ae = ToFAutoencoder(tof_dim) if USE_TOF else None\n        self.tof_proj = nn.Linear(8, D_MODEL) if USE_TOF else None\n\n        fusion_dim = D_MODEL * sum([USE_ACC, USE_ROT, USE_TOF])\n        self.head = nn.Sequential(\n            nn.LayerNorm(fusion_dim),\n            nn.Linear(fusion_dim, D_MODEL),\n            nn.GELU(),\n            nn.Dropout(DROPOUT),\n            nn.Linear(D_MODEL, num_classes)\n        )\n\n    def forward(self, acc, rot, tof):\n        feats = []\n\n        if USE_ACC:\n            feats.append(self.acc_enc(acc))\n        if USE_ROT:\n            feats.append(self.rot_enc(rot))\n        if USE_TOF:\n            _, z = self.tof_ae(tof)\n            feats.append(self.tof_proj(z))\n\n        z = torch.cat(feats, dim=1)\n        return self.head(z)\n\n# ============================================================\n# TRAIN\n# ============================================================\n\ndataset = HybridDataset(CSV_PATH)\nnum_classes = len(dataset.classes)\nacc_dim = dataset[0][\"acc\"].shape[1]\ntof_dim = dataset[0][\"tof\"].shape[0]\n\ntrain_size = int((1 - VAL_RATIO) * len(dataset))\ntrain_ds, val_ds = random_split(dataset, [train_size, len(dataset)-train_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\nval_loader = DataLoader(val_ds, batch_size=BATCH_SIZE)\n\nmodel = HybridModel(num_classes, acc_dim, tof_dim).to(DEVICE)\n\nif USE_TOF and FREEZE_TOF:\n    for p in model.tof_ae.encoder.parameters():\n        p.requires_grad = False\n\nopt = torch.optim.AdamW(model.parameters(), lr=LR)\nclf_loss = nn.CrossEntropyLoss()\ntof_loss = nn.MSELoss()\n\n# ============================================================\n# LOOP\n# ============================================================\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total = 0\n\n    for b in train_loader:\n        acc = b[\"acc\"].to(DEVICE)\n        rot = b[\"rot\"].to(DEVICE)\n        tof = b[\"tof\"].to(DEVICE)\n        y = b[\"label\"].to(DEVICE)\n\n        opt.zero_grad()\n        logits = model(acc, rot, tof)\n        loss = clf_loss(logits, y)\n\n        if USE_TOF:\n            recon, _ = model.tof_ae(tof)\n            loss = loss + 0.1 * tof_loss(recon, tof)\n\n        loss.backward()\n        opt.step()\n        total += loss.item()\n\n    print(f\"Epoch {epoch+1} | Loss {total/len(train_loader):.4f}\")\n\n# ============================================================\n# EVAL\n# ============================================================\n\nmodel.eval()\ncorrect, total = 0, 0\nwith torch.no_grad():\n    for b in val_loader:\n        logits = model(\n            b[\"acc\"].to(DEVICE),\n            b[\"rot\"].to(DEVICE),\n            b[\"tof\"].to(DEVICE)\n        )\n        pred = logits.argmax(1)\n        correct += (pred.cpu() == b[\"label\"]).sum().item()\n        total += len(pred)\n\nprint(f\"\\nValidation accuracy: {correct/total:.3f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.661329Z","iopub.status.idle":"2026-02-02T17:51:47.661596Z","shell.execute_reply.started":"2026-02-02T17:51:47.661479Z","shell.execute_reply":"2026-02-02T17:51:47.661492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, (data, target) in enumerate(test_loader):\n    if i >= 3:  # Test 3 samples\n        break\n    \n    data = data.to(DEVICE)\n    with torch.no_grad():\n        output = model(data)\n        probs = F.softmax(output, dim=1)\n        pred = output.argmax(dim=1)\n        \n    true_label = dataset.classes[target.item()]\n    pred_label = dataset.classes[pred.item()]\n    confidence = probs[0, pred.item()].item()\n    print(pred.item())\n    print(f\"Sample {i+1}: True={true_label:10s} | Pred={pred_label:10s} | \"\n          f\"Confidence={confidence:.3f} | \"\n          f\"{'CORRECT' if pred.item() == target.item() else 'WRONG'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.663589Z","iopub.status.idle":"2026-02-02T17:51:47.664224Z","shell.execute_reply.started":"2026-02-02T17:51:47.664078Z","shell.execute_reply":"2026-02-02T17:51:47.66409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def save_model_for_submission(model, label_encoder, feature_dim=16):\n    \"\"\"\n    Save the trained model and label encoder for Kaggle submission.\n    \n    Args:\n        model: Trained PyTorch model\n        label_encoder: Fitted LabelEncoder\n        feature_dim: Number of input features (should be 16)\n    \"\"\"\n    import os\n    import joblib\n    \n    # Create output directory\n    os.makedirs(\"/kaggle/working\", exist_ok=True)\n    \n    # 1. Save model weights\n    model_path = \"/kaggle/working/inceptiontime_model.pth\"\n    torch.save(model.state_dict(), model_path)\n    print(f\"✅ Model saved to: {model_path}\")\n    \n    # 2. Save label encoder\n    encoder_path = \"/kaggle/working/label_encoder.pkl\"\n    joblib.dump(label_encoder, encoder_path)\n    print(f\"✅ Label encoder saved to: {encoder_path}\")\n    \n    # 3. Save metadata\n    metadata = {\n        'feature_dim': feature_dim,\n        'window_before': WINDOW_BEFORE,\n        'window_after': WINDOW_AFTER,\n        'window_len': WINDOW_LEN,\n        'classes': label_encoder.classes_.tolist(),\n        'num_classes': len(label_encoder.classes_)\n    }\n    metadata_path = \"/kaggle/working/model_metadata.pkl\"\n    joblib.dump(metadata, metadata_path)\n    print(f\"✅ Metadata saved to: {metadata_path}\")\n    \n    # 4. Create a requirements.txt for reproducibility\n    requirements = \"\"\"torch>=2.0.0\nnumpy>=1.24.0\npandas>=2.0.0\npolars>=0.19.0\nscikit-learn>=1.3.0\njoblib>=1.3.0\n\"\"\"\n    \n    with open(\"/kaggle/working/requirements.txt\", \"w\") as f:\n        f.write(requirements)\n    print(f\"✅ Requirements file saved to: /kaggle/working/requirements.txt\")\nsave_model_for_submission(model, dataset.le, feature_dim=16)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.66518Z","iopub.status.idle":"2026-02-02T17:51:47.665945Z","shell.execute_reply.started":"2026-02-02T17:51:47.665766Z","shell.execute_reply":"2026-02-02T17:51:47.665782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(dataset.classes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.666804Z","iopub.status.idle":"2026-02-02T17:51:47.667079Z","shell.execute_reply.started":"2026-02-02T17:51:47.666934Z","shell.execute_reply":"2026-02-02T17:51:47.66695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nimport polars as pl\n\nimport kaggle_evaluation.cmi_inference_server\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    # Replace this function with your inference code.\n    # You can return either a Pandas or Polars dataframe, though Polars is recommended.\n    # Each prediction (except the very first) must be returned within 30 minutes of the batch features being provided.\n    classes = ['Above ear - pull hair' 'Cheek - pinch skin' 'Drink from bottle/cup'\n 'Eyebrow - pull hair' 'Eyelash - pull hair'\n 'Feel around in tray and pull out an object' 'Forehead - pull hairline'\n 'Forehead - scratch' 'Glasses on/off' 'Neck - pinch skin'\n 'Neck - scratch' 'Pinch knee/leg skin' 'Pull air toward your face'\n 'Scratch knee/leg skin' 'Text on phone' 'Wave hello' 'Write name in air'\n 'Write name on leg']\n    data = sequence.to(DEVICE)\n    with torch.no_grad():\n        output = model(data)\n        pred = output.argmax(dim=1)\n        \n    pred_label = classes[pred.item()]\n    return pred_label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.668412Z","iopub.status.idle":"2026-02-02T17:51:47.668646Z","shell.execute_reply.started":"2026-02-02T17:51:47.668541Z","shell.execute_reply":"2026-02-02T17:51:47.668552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T17:51:47.669406Z","iopub.status.idle":"2026-02-02T17:51:47.669609Z","shell.execute_reply.started":"2026-02-02T17:51:47.669513Z","shell.execute_reply":"2026-02-02T17:51:47.669522Z"}},"outputs":[],"execution_count":null}]}