{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":13242446,"sourceType":"datasetVersion","datasetId":8390817}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport os\n\n\n# Load preprocessed data (replace with your dataset path if needed)\ntrain_df = pd.read_pickle(\"//kaggle/input/processed-data/processed_train.pkl\")\ntest_df = pd.read_pickle(\"/kaggle/input/processed-data/processed_test.pkl\")\n\n# Convert all numeric columns to float32 to save memory\nnumeric_cols_train = train_df.select_dtypes(include=['float64', 'int64']).columns\ntrain_df[numeric_cols_train] = train_df[numeric_cols_train].astype('float32')\n\nnumeric_cols_test = test_df.select_dtypes(include=['float64', 'int64']).columns\ntest_df[numeric_cols_test] = test_df[numeric_cols_test].astype('float32')\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-06T04:14:19.433995Z","iopub.execute_input":"2025-10-06T04:14:19.434163Z","iopub.status.idle":"2025-10-06T04:14:52.869826Z","shell.execute_reply.started":"2025-10-06T04:14:19.434147Z","shell.execute_reply":"2025-10-06T04:14:52.869016Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Prepare Data for Time-Series Modeling\n\n\nFor time-series models, we need to organize the data into a sequence format where each sequence is a sliding window of sensor data.\n\nTime Series Windowing: This step involves converting the data into sequences of fixed lengths (e.g., window size = 50, 100, 200) and ensuring that the sequence data is aligned with the target (behavior).","metadata":{}},{"cell_type":"code","source":"print(\"Train columns:\", train_df.columns.tolist())\nprint(\"Test columns:\", test_df.columns.tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T04:14:52.871830Z","iopub.execute_input":"2025-10-06T04:14:52.872267Z","iopub.status.idle":"2025-10-06T04:14:52.877000Z","shell.execute_reply.started":"2025-10-06T04:14:52.872247Z","shell.execute_reply":"2025-10-06T04:14:52.876262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================\n# Setup & Utilities (OOM-safe)\n# ==========================\nimport time, math, gc, sys\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\ntorch.backends.cudnn.benchmark = True\nif hasattr(torch, 'set_float32_matmul_precision'):\n    torch.set_float32_matmul_precision('high')\n\n# ---------- Target column ----------\nTARGET_COL = \"behavior\"   # use correct label column in train_df\n\n# ---------- Label encoder (train only) ----------\ndef build_label_encoder(train_df, target_col=TARGET_COL):\n    # Ensure labels are all strings\n    all_labels = pd.Index(train_df[target_col].astype(str)).unique()\n    label2id = {lbl: i for i, lbl in enumerate(sorted(all_labels))}\n    id2label = {i: lbl for lbl, i in label2id.items()}\n    return label2id, id2label\n\n# Use it\nlabel2id, id2label = build_label_encoder(train_df, TARGET_COL)\nN_CLASSES = len(label2id)\nprint(f\"Detected {N_CLASSES} classes.\")\n\n\n# ---------- Feature columns ----------\ndef get_feature_columns(train_df, test_df, target_col=TARGET_COL):\n    train_feats = train_df.drop(columns=[target_col], errors='ignore').select_dtypes(include=[np.number]).columns\n    test_feats  = test_df.select_dtypes(include=[np.number]).columns\n    common = sorted(set(train_feats).intersection(set(test_feats)))\n    if len(common) == 0:\n        raise ValueError(\"No common numeric features between train and test.\")\n    return common\n\nFEATURE_COLS = get_feature_columns(train_df, test_df, TARGET_COL)\nN_FEATURES = len(FEATURE_COLS)\nprint(f\"Using {N_FEATURES} numeric features common to train/test.\")\n\n# ==========================\n# Positional Encoding (sinusoidal)\n# ==========================\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_len=4096):\n        super().__init__()\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2, dtype=torch.float32) * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position*div_term)\n        pe[:, 1::2] = torch.cos(position*div_term)\n        self.register_buffer('pe', pe)  # (max_len, d_model)\n\n    def forward(self, x):  # x: (B, T, D)\n        T = x.size(1)\n        return x + self.pe[:T].unsqueeze(0)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T04:14:52.877915Z","iopub.execute_input":"2025-10-06T04:14:52.878160Z","iopub.status.idle":"2025-10-06T04:14:57.393031Z","shell.execute_reply.started":"2025-10-06T04:14:52.878141Z","shell.execute_reply":"2025-10-06T04:14:57.392369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================\n# Models\n# ==========================\n\n# 4) TCN (Temporal Convolutional Network)\nclass Chomp1d(nn.Module):\n    def __init__(self, chomp_size): super().__init__(); self.chomp_size = chomp_size\n    def forward(self, x): return x[:, :, :-self.chomp_size].contiguous()\n\nclass TemporalBlock(nn.Module):\n    def __init__(self, in_ch, out_ch, kernel_size, stride, dilation, padding, dropout):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_ch, out_ch, kernel_size, stride=stride, padding=padding, dilation=dilation)\n        self.chomp1 = Chomp1d(padding)\n        self.relu1 = nn.ReLU()\n        self.drop1 = nn.Dropout(dropout)\n        self.conv2 = nn.Conv1d(out_ch, out_ch, kernel_size, stride=stride, padding=padding, dilation=dilation)\n        self.chomp2 = Chomp1d(padding)\n        self.relu2 = nn.ReLU()\n        self.drop2 = nn.Dropout(dropout)\n        self.downsample = nn.Conv1d(in_ch, out_ch, 1) if in_ch != out_ch else None\n        self.relu = nn.ReLU()\n\n    def forward(self, x):\n        out = self.conv1(x); out = self.chomp1(out); out = self.relu1(out); out = self.drop1(out)\n        out = self.conv2(out); out = self.chomp2(out); out = self.relu2(out); out = self.drop2(out)\n        res = x if self.downsample is None else self.downsample(x)\n        return self.relu(out + res)\n\nclass TCNClassifier(nn.Module):\n    def __init__(self, n_features, n_classes, channels=(64, 64, 128), kernel_size=5, dropout=0.1):\n        super().__init__()\n        layers = []\n        in_ch = n_features\n        dilation = 1\n        for out_ch in channels:\n            layers.append(TemporalBlock(in_ch, out_ch, kernel_size, stride=1,\n                                        dilation=dilation, padding=(kernel_size-1)*dilation, dropout=dropout))\n            in_ch = out_ch\n            dilation *= 2\n        self.net = nn.Sequential(*layers)\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        self.head = nn.Linear(in_ch, n_classes)\n\n    def forward(self, x):  # (B, T, F) -> (B, F, T)\n        z = self.net(x.transpose(1, 2))\n        z = self.pool(z).squeeze(-1)\n        return self.head(z)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T04:14:57.393794Z","iopub.execute_input":"2025-10-06T04:14:57.394012Z","iopub.status.idle":"2025-10-06T04:14:57.403121Z","shell.execute_reply.started":"2025-10-06T04:14:57.393994Z","shell.execute_reply":"2025-10-06T04:14:57.402493Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1) Vanilla Transformer (tokens=time steps)\nclass TransformerClassifier(nn.Module):\n    def __init__(self, n_features, n_classes, d_model=128, nhead=4, num_layers=2, dim_feedforward=256, dropout=0.1, pool='mean'):\n        super().__init__()\n        self.input_proj = nn.Linear(n_features, d_model)\n        self.pos = PositionalEncoding(d_model)\n        enc_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward,\n                                               dropout=dropout, batch_first=True)\n        self.encoder = nn.TransformerEncoder(enc_layer, num_layers=num_layers)\n        self.pool = pool\n        self.norm = nn.LayerNorm(d_model)\n        self.head = nn.Linear(d_model, n_classes)\n\n    def forward(self, x):  # x: (B, T, F)\n        z = self.input_proj(x)\n        z = self.pos(z)\n        z = self.encoder(z)\n        z = self.norm(z)\n        if self.pool == 'cls':\n            z = z[:, 0, :]\n        else:\n            z = z.mean(dim=1)\n        return self.head(z)\n\n\n# 2) TST (Time Series Transformer) style: InstanceNorm on features + Transformer\nclass TSTClassifier(nn.Module):\n    def __init__(self, n_features, n_classes, d_model=128, nhead=4, num_layers=2, dim_feedforward=256, dropout=0.1):\n        super().__init__()\n        self.inorm = nn.InstanceNorm1d(n_features, affine=False)  # normalize per-sample feature channels\n        self.input_proj = nn.Linear(n_features, d_model)\n        self.pos = PositionalEncoding(d_model)\n        enc = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward,\n                                         dropout=dropout, batch_first=True)\n        self.encoder = nn.TransformerEncoder(enc, num_layers=num_layers)\n        self.norm = nn.LayerNorm(d_model)\n        self.head = nn.Linear(d_model, n_classes)\n\n    def forward(self, x):  # (B, T, F)\n        # InstanceNorm expects (B, C, T)\n        xn = self.inorm(x.transpose(1, 2)).transpose(1, 2)\n        z = self.input_proj(xn)\n        z = self.pos(z)\n        z = self.encoder(z)\n        z = self.norm(z.mean(dim=1))\n        return self.head(z)\n\n# 3) Informer-Lite: Downsample sequence (distillation) + Transformer encoder\nclass InformerLiteClassifier(nn.Module):\n    def __init__(self, n_features, n_classes, d_model=128, nhead=4, num_layers=2, dim_feedforward=256, dropout=0.1, ds_factor=4):\n        super().__init__()\n        self.ds = nn.Conv1d(n_features, n_features, kernel_size=ds_factor, stride=ds_factor, groups=n_features)  # depthwise\n        self.input_proj = nn.Linear(n_features, d_model)\n        self.pos = PositionalEncoding(d_model)\n        enc = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward,\n                                         dropout=dropout, batch_first=True)\n        self.encoder = nn.TransformerEncoder(enc, num_layers=num_layers)\n        self.norm = nn.LayerNorm(d_model)\n        self.head = nn.Linear(d_model, n_classes)\n\n    def forward(self, x):  # (B, T, F)\n        # downsample along T via depthwise 1D conv on channels: need shape (B, C, T)\n        xt = x.transpose(1, 2)                       # (B, F, T)\n        xd = self.ds(xt).transpose(1, 2)             # (B, T', F)\n        z = self.input_proj(xd)\n        z = self.pos(z)\n        z = self.encoder(z)\n        z = self.norm(z.mean(dim=1))\n        return self.head(z)\n\n# 4) TCN (Temporal Convolutional Network)\nclass Chomp1d(nn.Module):\n    def __init__(self, chomp_size): super().__init__(); self.chomp_size = chomp_size\n    def forward(self, x): return x[:, :, :-self.chomp_size].contiguous()\n\nclass TemporalBlock(nn.Module):\n    def __init__(self, in_ch, out_ch, kernel_size, stride, dilation, padding, dropout):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_ch, out_ch, kernel_size, stride=stride, padding=padding, dilation=dilation)\n        self.chomp1 = Chomp1d(padding)\n        self.relu1 = nn.ReLU()\n        self.drop1 = nn.Dropout(dropout)\n        self.conv2 = nn.Conv1d(out_ch, out_ch, kernel_size, stride=stride, padding=padding, dilation=dilation)\n        self.chomp2 = Chomp1d(padding)\n        self.relu2 = nn.ReLU()\n        self.drop2 = nn.Dropout(dropout)\n        self.downsample = nn.Conv1d(in_ch, out_ch, 1) if in_ch != out_ch else None\n        self.relu = nn.ReLU()\n\n    def forward(self, x):\n        out = self.conv1(x); out = self.chomp1(out); out = self.relu1(out); out = self.drop1(out)\n        out = self.conv2(out); out = self.chomp2(out); out = self.relu2(out); out = self.drop2(out)\n        res = x if self.downsample is None else self.downsample(x)\n        return self.relu(out + res)\n\nclass TCNClassifier(nn.Module):\n    def __init__(self, n_features, n_classes, channels=(64, 64, 128), kernel_size=5, dropout=0.1):\n        super().__init__()\n        layers = []\n        in_ch = n_features\n        dilation = 1\n        for out_ch in channels:\n            layers.append(TemporalBlock(in_ch, out_ch, kernel_size, stride=1,\n                                        dilation=dilation, padding=(kernel_size-1)*dilation, dropout=dropout))\n            in_ch = out_ch\n            dilation *= 2\n        self.net = nn.Sequential(*layers)\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        self.head = nn.Linear(in_ch, n_classes)\n\n    def forward(self, x):  # (B, T, F) -> (B, F, T)\n        z = self.net(x.transpose(1, 2))\n        z = self.pool(z).squeeze(-1)\n        return self.head(z)\n\n","metadata":{}},{"cell_type":"code","source":"# ==========================\n# Training / Evaluation\n# ==========================\nmodel_name = 'vanilla_transformer'\n\n@torch.no_grad()\ndef macro_f1_from_logits(logits, targets, n_classes=N_CLASSES):\n    # logits: (N, C), targets: (N,)\n    preds = logits.argmax(dim=1).view(-1).cpu().numpy()\n    t = targets.view(-1).cpu().numpy()\n    # compute macro-F1 (no sklearn dependency)\n    f1s = []\n    for c in range(n_classes):\n        tp = np.sum((preds == c) & (t == c))\n        fp = np.sum((preds == c) & (t != c))\n        fn = np.sum((preds != c) & (t == c))\n        if tp == 0 and (fp+fn) == 0:\n            f1 = 1.0\n        else:\n            prec = tp / (tp + fp + 1e-12)\n            rec  = tp / (tp + fn + 1e-12)\n            f1 = 0.0 if (prec+rec)==0 else 2*prec*rec/(prec+rec)\n        f1s.append(f1)\n    return float(np.mean(f1s))\n\ndef count_params(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef train_one_epoch(model, loader, optimizer, scaler, criterion, grad_accum=1):\n    model.train()\n    total_loss = 0.0\n    n_seen = 0\n    for step, (x, y) in enumerate(loader):\n        x, y = x.to(DEVICE, non_blocking=True), y.to(DEVICE, non_blocking=True)\n        with torch.autocast(device_type='cuda' if DEVICE=='cuda' else 'cpu', dtype=torch.float16 if DEVICE=='cuda' else torch.bfloat16):\n            logits = model(x)\n            loss = criterion(logits, y) / grad_accum\n\n        scaler.scale(loss).backward()\n        if (step+1) % grad_accum == 0:\n            scaler.step(optimizer); scaler.update(); optimizer.zero_grad(set_to_none=True)\n\n        total_loss += loss.item() * grad_accum * x.size(0)\n        n_seen += x.size(0)\n\n        # small leak guard\n        if (step+1) % 200 == 0:\n            torch.cuda.empty_cache() if DEVICE=='cuda' else None\n    return total_loss / max(n_seen,1)\n\n@torch.no_grad()\ndef evaluate(model, loader, criterion, n_classes=N_CLASSES):\n    \"\"\"\n    Evaluate a model on a DataLoader and compute:\n    - Average loss\n    - Macro F1 score\n    - Accuracy (binary or multi-class)\n    \n    Args:\n        model: PyTorch model\n        loader: DataLoader with (x, y)\n        criterion: loss function (e.g., CrossEntropyLoss)\n        n_classes: number of classes\n    \n    Returns:\n        val_loss: average loss\n        macro_f1: macro F1 score\n        val_acc: accuracy\n        binary_flag: 'Yes' if binary, 'No' otherwise\n    \"\"\"\n    model.eval()\n    total_loss = 0.0\n    n_seen = 0\n    all_logits, all_targets = [], []\n\n    for x, y in loader:\n        x, y = x.to(DEVICE, non_blocking=True), y.to(DEVICE, non_blocking=True)\n        logits = model(x)\n        loss = criterion(logits, y)\n        total_loss += loss.item() * x.size(0)\n        n_seen += x.size(0)\n        all_logits.append(logits)\n        all_targets.append(y)\n\n    logits = torch.cat(all_logits, dim=0)\n    targets = torch.cat(all_targets, dim=0)\n\n    # Macro F1\n    preds = logits.argmax(dim=1)\n    f1_scores = []\n    for c in range(n_classes):\n        tp = ((preds == c) & (targets == c)).sum().item()\n        fp = ((preds == c) & (targets != c)).sum().item()\n        fn = ((preds != c) & (targets == c)).sum().item()\n        if tp == 0 and (fp + fn) == 0:\n            f1 = 1.0\n        else:\n            prec = tp / (tp + fp + 1e-12)\n            rec = tp / (tp + fn + 1e-12)\n            f1 = 0.0 if (prec + rec) == 0 else 2 * prec * rec / (prec + rec)\n        f1_scores.append(f1)\n    macro_f1 = float(np.mean(f1_scores))\n\n    # Accuracy\n    val_acc = (preds == targets).float().mean().item()\n\n    # Binary flag\n    binary_flag = 'Yes' if n_classes == 2 else 'No'\n\n    val_loss = total_loss / max(n_seen, 1)\n    return val_loss, macro_f1, val_acc, binary_flag\n\n\nimport torch\nfrom torch.utils.data import Dataset\n\n# Ensure this class is defined first\n# Make sure SequenceDataset is defined first\nclass SequenceDataset(Dataset):\n    def __init__(self, df, window_size, stride=1, target_col=TARGET_COL, feature_cols=FEATURE_COLS, label2id=label2id):\n        # Drop rows with NaN in target\n        self.df = df.dropna(subset=[target_col]).reset_index(drop=True)\n        self.feature_df = self.df[feature_cols]\n\n        # Map labels safely; unknown labels become -1\n        self.target = self.df[target_col].map(label2id).fillna(-1).astype(int).values\n\n        # Remove rows where target is -1 (unknown labels)\n        valid_idx = self.target != -1\n        self.feature_df = self.feature_df.iloc[valid_idx].reset_index(drop=True)\n        self.target = self.target[valid_idx]\n\n        self.window = int(window_size)\n        self.stride = int(stride)\n        self.n = max((len(self.feature_df) - self.window), 0) // self.stride\n\n    def __len__(self):\n        return self.n\n\n    def __getitem__(self, idx):\n        i = idx * self.stride\n        x = self.feature_df.iloc[i:i+self.window].to_numpy(dtype=np.float32)\n        y = self.target[i+self.window]\n        return torch.from_numpy(x), torch.tensor(y, dtype=torch.long)\n\nclass SequenceDatasetTest(Dataset):\n    def __init__(self, df, window_size, stride=1, feature_cols=FEATURE_COLS):\n        self.df = df.reset_index(drop=True)\n        self.feature_df = self.df[feature_cols]\n\n        self.window = int(window_size)\n        self.stride = int(stride)\n        self.n = max((len(self.feature_df) - self.window), 0) // self.stride\n\n    def __len__(self):\n        return self.n\n\n    def __getitem__(self, idx):\n        i = idx * self.stride\n        x = self.feature_df.iloc[i:i+self.window].to_numpy(dtype=np.float32)\n        return torch.from_numpy(x)\n\n# Now your run_experiment function can use SequenceDatasetTest\nimport os  # Make sure to import os\n\n# ==========================\n# Run Experiment (No Checkpoints)\n# ==========================\ndef run_experiment(model_name, model_ctor, window_size=50, stride=5,\n                   batch_size=64, epochs=5, lr=1e-3, grad_accum=1, num_workers=0):\n    print(f\"\\n=== {model_name} | window={window_size} stride={stride} ===\")\n    \n    # Initialize the dataset and dataloaders\n    train_ds = SequenceDataset(train_df, window_size, stride=stride,\n                               target_col=TARGET_COL, feature_cols=FEATURE_COLS, label2id=label2id)\n    test_ds  = SequenceDatasetTest(test_df, window_size, stride=stride, feature_cols=FEATURE_COLS)\n\n    if len(train_ds) == 0 or len(test_ds) == 0:\n        print(\"Dataset too small for given window/stride. Skipping.\")\n        return None\n\n    train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True,\n                              num_workers=num_workers, pin_memory=True)\n    test_loader  = DataLoader(test_ds, batch_size=batch_size, shuffle=False,\n                              num_workers=num_workers, pin_memory=True)\n\n    # Initialize the model\n    model = model_ctor().to(DEVICE)\n    print(f\"Params: {count_params(model):,}\")\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\n    criterion = nn.CrossEntropyLoss()\n    scaler = torch.amp.GradScaler(enabled=(DEVICE=='cuda'))\n\n    best_f1, history = -1.0, []\n    \n    # Training loop\n    for ep in range(epochs):\n        t1 = time.time()\n        train_loss = train_one_epoch(model, train_loader, optimizer, scaler, criterion, grad_accum=grad_accum)\n        \n        val_loss, val_f1, val_acc, binary_flag = evaluate(model, train_loader, criterion, n_classes=N_CLASSES)\n\n        dt = time.time() - t1\n        best_f1 = max(best_f1, val_f1)\n        history.append({'epoch': ep+1,\n                        'train_loss': train_loss,\n                        'val_loss': val_loss,\n                        'val_macro_f1': val_f1,\n                        'val_acc': val_acc,\n                        'Binary': binary_flag,\n                        'sec': dt})\n        print(f\"Epoch {ep+1:02d} | train {train_loss:.4f} | val {val_loss:.4f} | \"\n          f\"F1 {val_f1:.4f} | Acc {val_acc:.4f} | Binary {binary_flag} | {dt:.1f}s\")\n    \n    # Return the best results and history\n    result = {\n        'model': model_name,\n        'window': window_size,\n        'stride': stride,\n        'params': count_params(model),\n        'best_macro_f1': round(best_f1, 6),\n        'val_acc': round(val_acc, 6),  # <-- add accuracy\n        'history': history\n    }\n    return result\n\n# ==========================\n# Model registry (easy switch)\n# ==========================\n\ndef make_tcn():\n    return TCNClassifier(n_features=N_FEATURES, n_classes=N_CLASSES,\n                         channels=(64, 64, 128), kernel_size=5, dropout=0.1)\nMODEL_FACTORIES = {\n    'tcn': make_tcn,}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T04:14:57.403834Z","iopub.execute_input":"2025-10-06T04:14:57.404084Z","iopub.status.idle":"2025-10-06T04:14:57.456677Z","shell.execute_reply.started":"2025-10-06T04:14:57.404057Z","shell.execute_reply":"2025-10-06T04:14:57.456159Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"def make_tst():\n    return TSTClassifier(n_features=N_FEATURES, n_classes=N_CLASSES,\n                         d_model=128, nhead=4, num_layers=2, dim_feedforward=256, dropout=0.1)\n\ndef make_informer_lite():\n    return InformerLiteClassifier(n_features=N_FEATURES, n_classes=N_CLASSES,\n                                  d_model=128, nhead=4, num_layers=2, dim_feedforward=256, dropout=0.1, ds_factor=4)\n\ndef make_transformer():\n    return TransformerClassifier(n_features=N_FEATURES, n_classes=N_CLASSES,\n                                 d_model=128, nhead=4, num_layers=2, dim_feedforward=256, dropout=0.1)\n\nMODEL_FACTORIES = {\n\n    'vanilla_transformer': make_transformer,\n    'tst': make_tst,\n    'informer_lite': make_informer_lite,\n    \n}\n","metadata":{}},{"cell_type":"code","source":"# ==========================\n# Scaling window length experiments (OOM-safe defaults)\n# ==========================\nWINDOWS = [50]\nSTRIDE = 5           # reduce overlap to shrink dataset size ~5x\nBATCH = 64           # keep small to avoid OOM\nEPOCHS = 3           # adjust as you like\nLR = 1e-3\nGRAD_ACCUM = 2       # set >1 if you must keep BATCH even smaller\nWORKERS = 0          # Kaggle often stable with 0 or 2; avoid big numbers on large DataFrames\n\nimport pandas as pd\nimport gc\nimport time\n\n# Only run Vanilla Transformer\nMODELS_TO_RUN = {'tcn': make_tcn}\n\nsummary_results = []\n\nfor w in WINDOWS:\n    for model_name, ctor in MODELS_TO_RUN.items():\n        # Measure start time for total train\n        start_total = time.time()\n        out = run_experiment(\n            model_name=model_name,\n            model_ctor=ctor,\n            window_size=w,\n            stride=STRIDE,\n            batch_size=BATCH,\n            epochs=EPOCHS,\n            lr=LR,\n            grad_accum=GRAD_ACCUM,\n            num_workers=WORKERS\n        )\n        total_time = time.time() - start_total\n\n        if out is not None:\n            # Add inference time measurement (single forward pass)\n            model = ctor().to(DEVICE)\n            x_sample = next(iter(DataLoader(SequenceDataset(train_df, w, stride=STRIDE,\n                                                             target_col=TARGET_COL, \n                                                             feature_cols=FEATURE_COLS, \n                                                             label2id=label2id),\n                                             batch_size=1)))\n            x_input, _ = x_sample\n            x_input = x_input.to(DEVICE)\n            torch.cuda.synchronize() if DEVICE=='cuda' else None\n            start_infer = time.time()\n            _ = model(x_input)\n            torch.cuda.synchronize() if DEVICE=='cuda' else None\n            inference_time = time.time() - start_infer\n\n            # Detect binary classification (2 classes) or multi-class\n            binary_flag = 'Yes' if N_CLASSES == 2 else 'No'\n\n            summary_results.append({\n                'model_name': out['model'],\n                'runtime_id': f\"{out['model']}_w{out['window']}_s{out['stride']}\",\n                'notebook': 'Notebook 3',\n                'features_used': 'All',\n                'window_size': out['window'],\n                'optimizer/solver': 'AdamW',\n                'params (M)': out['params'] / 1e6,\n                'Binary': binary_flag,  # Added Binary flag\n                'Macro': out['best_macro_f1'],\n                'Final Score': out['best_macro_f1'],\n                'val_acc': out.get('val_acc', None),\n                'train_time': total_time,\n                'inference_time': inference_time\n            })\n\n        # free memory aggressively\n        torch.cuda.empty_cache() if DEVICE=='cuda' else None\n        gc.collect()\n\n# Create DataFrame\nfinal_summary_df = pd.DataFrame(summary_results)\nfinal_summary_df = final_summary_df.sort_values(['window_size', 'model_name']).reset_index(drop=True)\n\n# Display final table\npd.set_option('display.max_columns', None)\nprint(\"\\n=== Final Model Summary ===\")\ndisplay(final_summary_df)\n\n# Optional: Print binary result summary\nbinary_results = final_summary_df[['model_name', 'Binary', 'val_acc', 'Macro']]\nprint(\"\\n=== Binary Classification Results ===\")\ndisplay(binary_results)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T04:14:57.457361Z","iopub.execute_input":"2025-10-06T04:14:57.457567Z","iopub.status.idle":"2025-10-06T07:34:34.565055Z","shell.execute_reply.started":"2025-10-06T04:14:57.457544Z","shell.execute_reply":"2025-10-06T07:34:34.564456Z"}},"outputs":[],"execution_count":null}]}