{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12101487,"sourceType":"datasetVersion","datasetId":7618540},{"sourceId":242954653,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# PyTorch Implementation of https://www.kaggle.com/code/samithsachidanandan/lb-0-72-cmi-2025-tensor-flow\n\nimport os, json, joblib, numpy as np, pandas as pd\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nimport polars as pl\n# Configuration\nTRAIN = False                     # ← set to True when you want to train\nRAW_DIR = Path(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\")\nPRETRAINED_DIR = Path(\"/kaggle/input/cmi-exp1\")  # used when TRAIN=False\nEXPORT_DIR = Path(\"./\")                                    # artefacts will be saved here\nBATCH_SIZE = 64\nPAD_PERCENTILE = 95\nLR_INIT = 1e-3\nWD = 3e-3\nMIXUP_ALPHA = 0.4\nEPOCHS = 2\nPATIENCE = 40\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"▶ imports ready · pytorch {torch.__version__} · device: {device}\")\n\n# ================================\n# Model Components\n# ================================\n\nclass SEBlock(nn.Module):\n    def __init__(self, channels, reduction=8):\n        super().__init__()\n        self.squeeze = nn.AdaptiveAvgPool1d(1)\n        self.excitation = nn.Sequential(\n            nn.Linear(channels, channels // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(channels // reduction, channels, bias=False),\n            nn.Sigmoid()\n        )\n    \n    def forward(self, x):\n        b, c, _ = x.size()\n        y = self.squeeze(x).view(b, c)\n        y = self.excitation(y).view(b, c, 1)\n        return x * y.expand_as(x)\n\nclass ResidualSECNNBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n        super().__init__()\n        \n        # First conv block\n        self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n        self.bn1 = nn.BatchNorm1d(out_channels)\n        \n        # Second conv block\n        self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n        \n        # SE block\n        self.se = SEBlock(out_channels)\n        \n        # Shortcut connection\n        self.shortcut = nn.Sequential()\n        if in_channels != out_channels:\n            self.shortcut = nn.Sequential(\n                nn.Conv1d(in_channels, out_channels, 1, bias=False),\n                nn.BatchNorm1d(out_channels)\n            )\n        \n        self.pool = nn.MaxPool1d(pool_size)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        shortcut = self.shortcut(x)\n        \n        # First conv\n        out = F.relu(self.bn1(self.conv1(x)))\n        # Second conv\n        out = self.bn2(self.conv2(out))\n        \n        # SE block\n        out = self.se(out)\n        \n        # Add shortcut\n        out += shortcut\n        out = F.relu(out)\n        \n        # Pool and dropout\n        out = self.pool(out)\n        out = self.dropout(out)\n        \n        return out\n\nclass AttentionLayer(nn.Module):\n    def __init__(self, hidden_dim):\n        super().__init__()\n        self.attention = nn.Linear(hidden_dim, 1)\n        \n    def forward(self, x):\n        # x shape: (batch, seq_len, hidden_dim)\n        scores = torch.tanh(self.attention(x))  # (batch, seq_len, 1)\n        weights = F.softmax(scores.squeeze(-1), dim=1)  # (batch, seq_len)\n        context = torch.sum(x * weights.unsqueeze(-1), dim=1)  # (batch, hidden_dim)\n        return context\n\nclass TwoBranchModel(nn.Module):\n    def __init__(self, pad_len, imu_dim, tof_dim, n_classes, weight_decay=1e-4):\n        super().__init__()\n        self.imu_dim = imu_dim\n        self.tof_dim = tof_dim\n        \n        # IMU deep branch\n        self.imu_block1 = ResidualSECNNBlock(imu_dim, 64, 3, dropout=0.3, weight_decay=weight_decay)\n        self.imu_block2 = ResidualSECNNBlock(64, 128, 5, dropout=0.3, weight_decay=weight_decay)\n        \n        # TOF/Thermal lighter branch\n        self.tof_conv1 = nn.Conv1d(tof_dim, 64, 3, padding=1, bias=False)\n        self.tof_bn1 = nn.BatchNorm1d(64)\n        self.tof_pool1 = nn.MaxPool1d(2)\n        self.tof_drop1 = nn.Dropout(0.3)\n        \n        self.tof_conv2 = nn.Conv1d(64, 128, 3, padding=1, bias=False)\n        self.tof_bn2 = nn.BatchNorm1d(128)\n        self.tof_pool2 = nn.MaxPool1d(2)\n        self.tof_drop2 = nn.Dropout(0.3)\n        \n        # BiLSTM\n        self.bilstm = nn.LSTM(256, 128, bidirectional=True, batch_first=True)\n        self.lstm_dropout = nn.Dropout(0.4)\n        \n        # Attention\n        self.attention = AttentionLayer(256)  # 128*2 for bidirectional\n        \n        # Dense layers\n        self.dense1 = nn.Linear(256, 256, bias=False)\n        self.bn_dense1 = nn.BatchNorm1d(256)\n        self.drop1 = nn.Dropout(0.5)\n        \n        self.dense2 = nn.Linear(256, 128, bias=False)\n        self.bn_dense2 = nn.BatchNorm1d(128)\n        self.drop2 = nn.Dropout(0.3)\n        \n        self.classifier = nn.Linear(128, n_classes)\n        \n    def forward(self, x):\n        # Split input\n        imu = x[:, :, :self.imu_dim].transpose(1, 2)  # (batch, imu_dim, seq_len)\n        tof = x[:, :, self.imu_dim:].transpose(1, 2)  # (batch, tof_dim, seq_len)\n        \n        # IMU branch\n        x1 = self.imu_block1(imu)\n        x1 = self.imu_block2(x1)\n        \n        # TOF branch\n        x2 = F.relu(self.tof_bn1(self.tof_conv1(tof)))\n        x2 = self.tof_drop1(self.tof_pool1(x2))\n        x2 = F.relu(self.tof_bn2(self.tof_conv2(x2)))\n        x2 = self.tof_drop2(self.tof_pool2(x2))\n        \n        # Concatenate branches\n        merged = torch.cat([x1, x2], dim=1).transpose(1, 2)  # (batch, seq_len, 256)\n        \n        # BiLSTM\n        lstm_out, _ = self.bilstm(merged)\n        lstm_out = self.lstm_dropout(lstm_out)\n        \n        # Attention\n        attended = self.attention(lstm_out)\n        \n        # Dense layers\n        x = F.relu(self.bn_dense1(self.dense1(attended)))\n        x = self.drop1(x)\n        x = F.relu(self.bn_dense2(self.dense2(x)))\n        x = self.drop2(x)\n        \n        # Classification\n        logits = self.classifier(x)\n        return logits\n\n# ================================\n# Data Handling\n# ================================\n\ndef pad_sequences_torch(sequences, maxlen, padding='post', truncating='post', value=0.0):\n    \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n    result = []\n    for seq in sequences:\n        if len(seq) >= maxlen:\n            if truncating == 'post':\n                seq = seq[:maxlen]\n            else:  # 'pre'\n                seq = seq[-maxlen:]\n        else:\n            pad_len = maxlen - len(seq)\n            if padding == 'post':\n                seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n            else:  # 'pre'\n                seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n        result.append(seq)\n    return np.array(result, dtype=np.float32)\n\ndef preprocess_sequence(df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n    \"\"\"Normalizes and cleans the time series sequence\"\"\"\n    mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n    return scaler.transform(mat).astype('float32')\n\nclass MixupDataset(Dataset):\n    \"\"\"Dataset with MixUp augmentation\"\"\"\n    def __init__(self, X, y, alpha=0.2):\n        self.X = torch.FloatTensor(X)\n        self.y = torch.FloatTensor(y)\n        self.alpha = alpha\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        x, y = self.X[idx], self.y[idx]\n        \n        if self.alpha > 0:\n            lam = np.random.beta(self.alpha, self.alpha)\n            rand_idx = np.random.randint(0, len(self.X))\n            x_rand, y_rand = self.X[rand_idx], self.y[rand_idx]\n            \n            x = lam * x + (1 - lam) * x_rand\n            y = lam * y + (1 - lam) * y_rand\n            \n        return x, y\n\nclass EarlyStopping:\n    \"\"\"Early stopping utility\"\"\"\n    def __init__(self, patience=7, min_delta=0, restore_best_weights=True):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.restore_best_weights = restore_best_weights\n        self.best_loss = None\n        self.counter = 0\n        self.best_weights = None\n        \n    def __call__(self, val_loss, model):\n        if self.best_loss is None:\n            self.best_loss = val_loss\n            self.save_checkpoint(model)\n        elif val_loss < self.best_loss - self.min_delta:\n            self.best_loss = val_loss\n            self.counter = 0\n            self.save_checkpoint(model)\n        else:\n            self.counter += 1\n            \n        if self.counter >= self.patience:\n            if self.restore_best_weights:\n                model.load_state_dict(self.best_weights)\n            return True\n        return False\n    \n    def save_checkpoint(self, model):\n        self.best_weights = model.state_dict().copy()\n\ndef label_smoothing_loss(pred, target, smoothing=0.1):\n    \"\"\"Label smoothing loss\"\"\"\n    confidence = 1.0 - smoothing\n    log_probs = F.log_softmax(pred, dim=-1)\n    nll_loss = -log_probs.gather(dim=-1, index=target.unsqueeze(1))\n    nll_loss = nll_loss.squeeze(1)\n    smooth_loss = -log_probs.mean(dim=-1)\n    loss = confidence * nll_loss + smoothing * smooth_loss\n    return loss.mean()\n\n# ================================\n# Training Pipeline\n# ================================\n\nif TRAIN:\n    print(\"▶ TRAIN MODE – loading dataset …\")\n    df = pd.read_csv(RAW_DIR / \"train.csv\")\n\n    # Label encoding\n    le = LabelEncoder()\n    df['gesture_int'] = le.fit_transform(df['gesture'])\n    np.save(EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n\n    # Feature list\n    meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n                 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter'}\n    feature_cols = [c for c in df.columns if c not in meta_cols]\n\n    imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n    tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n    print(f\"  IMU {len(imu_cols)} | TOF/THM {len(tof_cols)} | total {len(feature_cols)} features\")\n\n    # Global scaler\n    scaler = StandardScaler().fit(df[feature_cols].ffill().bfill().fillna(0).values)\n    joblib.dump(scaler, EXPORT_DIR / \"scaler.pkl\")\n\n    # Build sequences\n    seq_gp = df.groupby('sequence_id')\n    X_list, y_list, lens = [], [], []\n    for seq_id, seq in seq_gp:\n        mat = preprocess_sequence(seq, feature_cols, scaler)\n        X_list.append(mat)\n        y_list.append(seq['gesture_int'].iloc[0])\n        lens.append(len(mat))\n    \n    pad_len = int(np.percentile(lens, PAD_PERCENTILE))\n    np.save(EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n    np.save(EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n\n    X = pad_sequences_torch(X_list, maxlen=pad_len, padding='post', truncating='post')\n    y = np.eye(len(le.classes_))[y_list].astype(np.float32)  # One-hot encoding\n\n    # Split\n    X_tr, X_val, y_tr, y_val = train_test_split(X, y, test_size=0.2, random_state=42, \n                                                stratify=y_list)\n\n    # Class weights\n    cw_vals = compute_class_weight('balanced', classes=np.arange(len(le.classes_)), y=y_list)\n    class_weight = torch.FloatTensor(cw_vals).to(device)\n\n    # Data loaders\n    train_dataset = MixupDataset(X_tr, y_tr, alpha=MIXUP_ALPHA)\n    val_dataset = MixupDataset(X_val, y_val, alpha=0.0)  # No mixup for validation\n    \n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    # Model\n    model = TwoBranchModel(pad_len, len(imu_cols), len(tof_cols), \n                          len(le.classes_), weight_decay=WD).to(device)\n    \n    # Optimizer and scheduler\n    optimizer = Adam(model.parameters(), lr=LR_INIT, weight_decay=WD)\n    steps_per_epoch = len(train_loader)\n    scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=5*steps_per_epoch)\n    \n    # Early stopping\n    early_stopping = EarlyStopping(patience=PATIENCE, restore_best_weights=True)\n    \n    # Training loop\n    print(\"▶ Starting training...\")\n    for epoch in range(EPOCHS):\n        # Training\n        model.train()\n        train_loss = 0.0\n        train_acc = 0.0\n        \n        for batch_x, batch_y in train_loader:\n            batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n            \n            optimizer.zero_grad()\n            logits = model(batch_x)\n            \n            # Handle mixup targets\n            if len(batch_y.shape) == 2 and batch_y.shape[1] > 1:  # One-hot or mixed\n                loss = -torch.sum(F.log_softmax(logits, dim=1) * batch_y, dim=1).mean()\n                # For accuracy, use argmax of target\n                targets = batch_y.argmax(dim=1)\n            else:\n                targets = batch_y.long()\n                loss = label_smoothing_loss(logits, targets, smoothing=0.1)\n            \n            # Apply class weights\n            if len(batch_y.shape) == 2:\n                sample_weights = torch.sum(batch_y * class_weight.unsqueeze(0), dim=1)\n                loss = (loss * sample_weights).mean()\n            \n            loss.backward()\n            optimizer.step()\n            scheduler.step()\n            \n            train_loss += loss.item()\n            train_acc += (logits.argmax(dim=1) == targets).float().mean().item()\n        \n        # Validation\n        model.eval()\n        val_loss = 0.0\n        val_acc = 0.0\n        \n        with torch.no_grad():\n            for batch_x, batch_y in val_loader:\n                batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n                logits = model(batch_x)\n                targets = batch_y.argmax(dim=1)\n                loss = F.cross_entropy(logits, targets)\n                val_loss += loss.item()\n                val_acc += (logits.argmax(dim=1) == targets).float().mean().item()\n        \n        train_loss /= len(train_loader)\n        train_acc /= len(train_loader)\n        val_loss /= len(val_loader)\n        val_acc /= len(val_loader)\n        \n        print(f\"Epoch {epoch+1}/{EPOCHS} - \"\n              f\"train_loss: {train_loss:.4f} - train_acc: {train_acc:.4f} - \"\n              f\"val_loss: {val_loss:.4f} - val_acc: {val_acc:.4f}\")\n        \n        # Early stopping\n        if early_stopping(val_loss, model):\n            print(f\"Early stopping at epoch {epoch+1}\")\n            break\n    \n    # Save model\n    torch.save({\n        'model_state_dict': model.state_dict(),\n        'imu_dim': len(imu_cols),\n        'tof_dim': len(tof_cols),\n        'n_classes': len(le.classes_),\n        'pad_len': pad_len\n    }, EXPORT_DIR / \"gesture_two_branch_mixup.pth\")\n    \n    print(\"✔ Training done – artefacts saved in\", EXPORT_DIR)\n\n    # Quick metric (if competition metric is available)\n    try:\n        from cmi_2025_metric_copy_for_import import CompetitionMetric\n        model.eval()\n        with torch.no_grad():\n            val_preds = []\n            val_targets = []\n            for batch_x, batch_y in val_loader:\n                batch_x = batch_x.to(device)\n                logits = model(batch_x)\n                val_preds.extend(logits.argmax(dim=1).cpu().numpy())\n                val_targets.extend(batch_y.argmax(dim=1).numpy())\n        \n        h_f1 = CompetitionMetric().calculate_hierarchical_f1(\n            pd.DataFrame({'gesture': le.classes_[val_targets]}),\n            pd.DataFrame({'gesture': le.classes_[val_preds]}))\n        print(\"Hold‑out H‑F1 =\", round(h_f1, 4))\n    except ImportError:\n        print(\"Competition metric not available\")\n\nelse:\n    print(\"▶ INFERENCE MODE – loading artefacts from\", PRETRAINED_DIR)\n    feature_cols = np.load(PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n    pad_len = int(np.load(PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n    scaler = joblib.load(PRETRAINED_DIR / \"scaler.pkl\")\n    gesture_classes = np.load(PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n\n    imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n    tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n\n    # Load model\n    checkpoint = torch.load(PRETRAINED_DIR / \"gesture_two_branch_mixup.pth\", map_location=device)\n    model = TwoBranchModel(\n        checkpoint['pad_len'], \n        checkpoint['imu_dim'], \n        checkpoint['tof_dim'], \n        checkpoint['n_classes']\n    ).to(device)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    model.eval()\n    print(\"  model, scaler, pads loaded – ready for evaluation\")\n\n# Make sure gesture_classes exists in both modes\nif TRAIN:\n    gesture_classes = le.classes_\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    \"\"\"Prediction function for Kaggle competition\"\"\"\n    global gesture_classes\n    if gesture_classes is None:\n        gesture_classes = np.load(PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n\n    df_seq = sequence.to_pandas()\n    mat = preprocess_sequence(df_seq, feature_cols, scaler)\n    pad = pad_sequences_torch([mat], maxlen=pad_len, padding='post', truncating='post')\n    \n    model.eval()\n    with torch.no_grad():\n        x = torch.FloatTensor(pad).to(device)\n        logits = model(x)\n        idx = int(logits.argmax(dim=1)[0].cpu().numpy())\n    \n    return str(gesture_classes[idx])\n\n# Kaggle competition interface\nimport kaggle_evaluation.cmi_inference_server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T21:43:50.874218Z","iopub.execute_input":"2025-06-08T21:43:50.874525Z","iopub.status.idle":"2025-06-08T21:43:51.940343Z","shell.execute_reply.started":"2025-06-08T21:43:50.874504Z","shell.execute_reply":"2025-06-08T21:43:51.939733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}