{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Challenge: advanced ConvNet training\n\n**Model.** CIFARNet is a residual CNN designed for 32×32 images (~7.8M parameters). It has a convolutional stem followed by three stages, each of which halves the spatial resolution (32→16→8→4) and doubles the channel count (128→256→512). Every stage ends with a residual block that combines two 3×3 convolutions with a Squeeze-and-Excitation module, so the network learns how much each channel should contribute before the skip connection adds the input back. The classifier concatenates global average pooling and global max pooling, then applies dropout and a linear layer.\n\n**Training.** The model is trained for 100 epochs with AdamW (lr 3e-3, weight decay 0.05). The learning rate warms up linearly over the first 5 epochs and then follows a cosine decay to zero. Besides random cropping and horizontal flips, half of the training batches use CutMix, which pastes a patch from one image onto another and mixes their labels in proportion to the patch area. CutMix is turned off for the final 5 epochs so the model finishes on clean images. Label smoothing (0.1) and mixed-precision training are also used.","metadata":{}},{"cell_type":"code","source":"import os, glob, re, time, math\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torchvision.transforms as T\nfrom PIL import Image\n\ndevice = torch.device('cuda')\nCIFAR_MEAN, CIFAR_STD = (0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616)\n\n# ---------- Data ----------\ntrain_tfms = T.Compose([\n    T.RandomCrop(32, padding=4, padding_mode='reflect'),\n    T.RandomHorizontalFlip(),\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\ntest_tfms = T.Compose([T.ToTensor(), T.Normalize(CIFAR_MEAN, CIFAR_STD)])\n\ntrain_ds = torchvision.datasets.CIFAR10('./data', train=True,  download=True, transform=train_tfms)\ntest_ds  = torchvision.datasets.CIFAR10('./data', train=False, download=True, transform=test_tfms)\nclasses = train_ds.classes\n\ntrain_dl = DataLoader(train_ds, batch_size=256, shuffle=True, num_workers=4,\n                      pin_memory=True, persistent_workers=True, drop_last=True)\ntest_dl  = DataLoader(test_ds, batch_size=512, shuffle=False, num_workers=4, pin_memory=True)\n\n# ---------- Custom network ----------\ndef conv_bn(cin, cout, pool=False):\n    layers = [nn.Conv2d(cin, cout, 3, padding=1, bias=False), nn.BatchNorm2d(cout), nn.ReLU(inplace=True)]\n    if pool: layers.append(nn.MaxPool2d(2))\n    return nn.Sequential(*layers)\n\nclass SE(nn.Module):\n    \"\"\"Squeeze-and-Excitation: learns a per-channel importance weight.\"\"\"\n    def __init__(self, c, r=16):\n        super().__init__()\n        self.gate = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Conv2d(c, c // r, 1), nn.ReLU(inplace=True),\n            nn.Conv2d(c // r, c, 1), nn.Sigmoid())\n    def forward(self, x):\n        return x * self.gate(x)\n\nclass ResSEBlock(nn.Module):\n    \"\"\"Two 3x3 convs + channel attention, with an identity skip connection.\"\"\"\n    def __init__(self, c):\n        super().__init__()\n        self.body = nn.Sequential(conv_bn(c, c), conv_bn(c, c), SE(c))\n    def forward(self, x):\n        return x + self.body(x)\n\nclass CIFARNet(nn.Module):\n    def __init__(self, num_classes=10):\n        super().__init__()\n        self.stem   = conv_bn(3, 64)                                          # 32x32\n        self.stage1 = nn.Sequential(conv_bn(64, 128, pool=True),  ResSEBlock(128))   # 16x16\n        self.stage2 = nn.Sequential(conv_bn(128, 256, pool=True), ResSEBlock(256))   # 8x8\n        self.stage3 = nn.Sequential(conv_bn(256, 512, pool=True), ResSEBlock(512))   # 4x4\n        self.head   = nn.Sequential(nn.Flatten(), nn.Dropout(0.2), nn.Linear(1024, num_classes))\n\n    def forward(self, x):\n        x = self.stage3(self.stage2(self.stage1(self.stem(x))))\n        # concat global avg-pool + global max-pool -> 1024 features\n        x = torch.cat([F.adaptive_avg_pool2d(x, 1), F.adaptive_max_pool2d(x, 1)], dim=1)\n        return self.head(x)\n\nmodel = CIFARNet().to(device)                 # random init, no pretrained weights\nprint(f'{sum(p.numel() for p in model.parameters()):,} parameters')\nif torch.cuda.device_count() > 1:             # use T4s\n    model = nn.DataParallel(model)\n\n# ---------- CutMix ----------\ndef cutmix(x, y, alpha=1.0):\n    lam = np.random.beta(alpha, alpha)\n    idx = torch.randperm(x.size(0), device=x.device)\n    H, W = x.shape[2], x.shape[3]\n    cut = math.sqrt(1 - lam)\n    ch, cw = int(H * cut), int(W * cut)\n    cy, cx = np.random.randint(H), np.random.randint(W)\n    y1, y2 = max(cy - ch // 2, 0), min(cy + ch // 2, H)\n    x1, x2 = max(cx - cw // 2, 0), min(cx + cw // 2, W)\n    x[:, :, y1:y2, x1:x2] = x[idx, :, y1:y2, x1:x2]\n    lam = 1 - (y2 - y1) * (x2 - x1) / (H * W)\n    return x, y, y[idx], lam\n\n# ---------- AdamW + warmup + cosine ----------\nepochs, warmup_epochs, no_mix_epochs = 100, 5, 5\nmax_lr, weight_decay = 3e-3, 0.05\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=max_lr, weight_decay=weight_decay)\ntotal_steps, warmup_steps = epochs * len(train_dl), warmup_epochs * len(train_dl)\n\ndef lr_lambda(step):\n    if step < warmup_steps:\n        return (step + 1) / warmup_steps\n    progress = (step - warmup_steps) / (total_steps - warmup_steps)\n    return 0.5 * (1 + math.cos(math.pi * progress))\n\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1)\nscaler = torch.amp.GradScaler('cuda')\n\n# ---------- Train ----------\nfor epoch in range(epochs):\n    model.train()\n    t0 = time.time()\n    use_cutmix = epoch < epochs - no_mix_epochs       # last 5 epochs: clean images only\n    for xb, yb in train_dl:\n        xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n        with torch.amp.autocast('cuda'):\n            if use_cutmix and np.random.rand() < 0.5:\n                xb, ya, yb2, lam = cutmix(xb, yb)\n                out = model(xb)\n                loss = lam * criterion(out, ya) + (1 - lam) * criterion(out, yb2)\n            else:\n                loss = criterion(model(xb), yb)\n        optimizer.zero_grad(set_to_none=True)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()\n\n    model.eval()\n    correct = total = 0\n    with torch.no_grad(), torch.amp.autocast('cuda'):\n        for xb, yb in test_dl:\n            xb, yb = xb.to(device), yb.to(device)\n            correct += (model(xb).argmax(1) == yb).sum().item()\n            total += yb.size(0)\n    print(f'Epoch {epoch+1}/{epochs}  lr: {scheduler.get_last_lr()[0]:.5f}  '\n          f'test_acc: {correct/total:.4f}  ({time.time()-t0:.1f}s)')\n\n# ---------- Final accuracy w/ TTA ----------\n@torch.no_grad()\ndef predict_tta(xb):\n    with torch.amp.autocast('cuda'):\n        return F.softmax(model(xb), 1) + F.softmax(model(torch.flip(xb, [3])), 1)\n\nmodel.eval()\ncorrect = total = 0\nfor xb, yb in test_dl:\n    xb, yb = xb.to(device), yb.to(device)\n    correct += (predict_tta(xb).argmax(1) == yb).sum().item()\n    total += yb.size(0)\nprint('Final test accuracy (TTA):', correct / total)\n\n# ---------- Kaggle submission ----------\nTEST_7Z_PATH, EXTRACT_DIR = '/kaggle/input/competitions/cifar-10/test.7z', './test'\n!pip -q install py7zr\nimport py7zr\nif not os.path.isdir(EXTRACT_DIR) or len(glob.glob(EXTRACT_DIR + '/*')) == 0:\n    with py7zr.SevenZipFile(TEST_7Z_PATH, mode='r') as archive:\n        archive.extractall(path='.')\nprint('Extracted files:', len(glob.glob(EXTRACT_DIR + '/*')))\n\nclass KaggleTestDataset(Dataset):\n    def __init__(self, folder, transform):\n        paths = glob.glob(os.path.join(folder, '*.png'))\n        self.paths = sorted(paths, key=lambda p: int(re.findall(r'\\d+', os.path.basename(p))[0]))\n        self.transform = transform\n    def __len__(self): return len(self.paths)\n    def __getitem__(self, i):\n        p = self.paths[i]\n        img_id = int(re.findall(r'\\d+', os.path.basename(p))[0])\n        return self.transform(Image.open(p).convert('RGB')), img_id\n\nkaggle_dl = DataLoader(KaggleTestDataset(EXTRACT_DIR, test_tfms), batch_size=512, shuffle=False, num_workers=4)\n\nids, preds = [], []\nfor xb, batch_ids in kaggle_dl:\n    preds.extend(predict_tta(xb.to(device)).argmax(1).cpu().tolist())\n    ids.extend(batch_ids.tolist())\n\nsubmission = pd.DataFrame({'id': ids, 'label': [classes[p] for p in preds]})\nsubmission.sort_values('id').to_csv('submission.csv', index=False)\nprint(submission.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T19:27:35.957282Z","iopub.execute_input":"2026-10-05T19:27:35.957596Z","iopub.status.idle":"2026-10-05T20:34:55.053063Z","shell.execute_reply.started":"2026-10-05T19:27:35.957572Z","shell.execute_reply":"2026-10-05T20:34:55.051681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_7Z_PATH = glob.glob('/kaggle/input/**/test.7z', recursive=True)[0]\nEXTRACT_DIR = './test'\nprint('Using:', TEST_7Z_PATH)\n\n!pip -q install py7zr\nimport py7zr\nif not os.path.isdir(EXTRACT_DIR) or len(glob.glob(EXTRACT_DIR + '/*')) == 0:\n    with py7zr.SevenZipFile(TEST_7Z_PATH, mode='r') as archive:\n        archive.extractall(path='.')\nprint('Extracted files:', len(glob.glob(EXTRACT_DIR + '/*')))\n\nclass KaggleTestDataset(Dataset):\n    def __init__(self, folder, transform):\n        paths = glob.glob(os.path.join(folder, '*.png'))\n        self.paths = sorted(paths, key=lambda p: int(re.findall(r'\\d+', os.path.basename(p))[0]))\n        self.transform = transform\n    def __len__(self): return len(self.paths)\n    def __getitem__(self, i):\n        p = self.paths[i]\n        img_id = int(re.findall(r'\\d+', os.path.basename(p))[0])\n        return self.transform(Image.open(p).convert('RGB')), img_id\n\nkaggle_dl = DataLoader(KaggleTestDataset(EXTRACT_DIR, test_tfms), batch_size=512, shuffle=False, num_workers=4)\n\nids, preds = [], []\nfor xb, batch_ids in kaggle_dl:\n    preds.extend(predict_tta(xb.to(device)).argmax(1).cpu().tolist())\n    ids.extend(batch_ids.tolist())\n\nsubmission = pd.DataFrame({'id': ids, 'label': [classes[p] for p in preds]})\nsubmission.sort_values('id').to_csv('submission.csv', index=False)\nprint(submission.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T20:44:30.602182Z","iopub.execute_input":"2026-10-05T20:44:30.602602Z","iopub.status.idle":"2026-10-05T20:49:28.991571Z","shell.execute_reply.started":"2026-10-05T20:44:30.602576Z","shell.execute_reply":"2026-10-05T20:49:28.990786Z"}},"outputs":[],"execution_count":null}]}