{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CIFAR-10: custom ResNet-SE · AdamW · warmup + cosine · CutMix · EMA\n\n* **Network:** my own design, built from scratch and randomly initialised. It has no pre-trained weights.\n  A ResNet9-style backbone (conv → max-pool downsampling) with extra residual blocks,\n  squeeze-and-excitation on every residual branch, zero-initialised residual BN, and an avg+max pooled head.\n* **Optimizer:** AdamW (decoupled weight decay; no decay on BN / bias).\n* **LR schedule:** linear warmup followed by cosine annealing, updated every step.\n* **Augmentation:** random crop (reflect pad 4) + horizontal flip + **CutMix**. All of it runs on the GPU.\n  CutMix is switched off for the last few epochs.\n* **Extras:** label smoothing, EMA of weights, mixed precision, flip test-time augmentation.\n\nSettings: Accelerator = GPU (T4 or P100). Internet is only needed if `7z` is missing (it then pip-installs `py7zr`).","metadata":{}},{"cell_type":"code","source":"import os, time, math, glob, shutil, subprocess, random, copy\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom PIL import Image\nfrom multiprocessing import Pool\n\nCFG = dict(\n    epochs=150,             # ~45-75 min on a T4/P100\n    batch_size=512,\n    peak_lr=2e-3,\n    min_lr_ratio=0.01,      # final lr = 1% of peak\n    warmup_epochs=5,\n    weight_decay=5e-2,\n    label_smoothing=0.1,\n    cutmix_prob=0.5,\n    cutmix_alpha=1.0,\n    no_mix_last_epochs=10,  # train the final epochs on un-mixed images\n    ema_decay=0.999,\n    val_size=5000,          # 5000 = hold out a validation split; set to 0 for the final run on all 50k images\n    seed=42,\n)\n\nCLASSES = ['airplane', 'automobile', 'bird', 'cat', 'deer',\n           'dog', 'frog', 'horse', 'ship', 'truck']\nCLASS_TO_IDX = {c: i for i, c in enumerate(CLASSES)}\n\ndef seed_everything(s):\n    random.seed(s); np.random.seed(s); torch.manual_seed(s); torch.cuda.manual_seed_all(s)\n\nseed_everything(CFG['seed'])\ndev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ntorch.backends.cudnn.benchmark = True\nprint('device:', dev, torch.cuda.get_device_name(0) if dev.type == 'cuda' else '')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:05:35.99448Z","iopub.execute_input":"2026-10-10T08:05:35.994804Z","iopub.status.idle":"2026-10-10T08:05:42.874544Z","shell.execute_reply.started":"2026-10-10T08:05:35.994782Z","shell.execute_reply":"2026-10-10T08:05:42.873603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Load the competition data (train.7z / test.7z → numpy arrays, cached)","metadata":{}},{"cell_type":"code","source":"WORK = '/tmp/cifar10'\nos.makedirs(WORK, exist_ok=True)\n\ndef find(name):\n    hits = glob.glob(f'/kaggle/input/**/{name}', recursive=True)\n    assert hits, f'{name} not found. Add the cifar-10 competition data to the notebook.'\n    return hits[0]\n\ndef extract(archive, dest_root, folder):\n    out = os.path.join(dest_root, folder)\n    if os.path.isdir(out) and len(os.listdir(out)) > 0:\n        return out\n    t = time.time()\n    if shutil.which('7z'):\n        subprocess.run(['7z', 'x', '-y', archive, f'-o{dest_root}'], check=True,\n                       stdout=subprocess.DEVNULL)\n    else:\n        subprocess.run(['pip', 'install', '-q', 'py7zr'], check=True)\n        import py7zr\n        with py7zr.SevenZipFile(archive, 'r') as z:\n            z.extractall(dest_root)\n    print(f'extracted {archive} in {time.time() - t:.0f}s')\n    return out\n\ndef load_png(path):\n    return np.asarray(Image.open(path).convert('RGB'), dtype=np.uint8)\n\ndef load_folder(folder):\n    n = len([f for f in os.listdir(folder) if f.endswith('.png')])\n    paths = [os.path.join(folder, f'{i}.png') for i in range(1, n + 1)]  # ids are 1..n\n    with Pool(os.cpu_count()) as p:\n        imgs = p.map(load_png, paths, chunksize=2000)\n    return np.stack(imgs)  # (n, 32, 32, 3) uint8\n\ndef cached(name, fn):\n    path = os.path.join(WORK, name)\n    if os.path.exists(path):\n        return np.load(path)\n    arr = fn(); np.save(path, arr); return arr\n\nt0 = time.time()\ntrain_dir = extract(find('train.7z'), WORK, 'train')\nX_all = cached('train_x.npy', lambda: load_folder(train_dir))\nlabels = pd.read_csv(find('trainLabels.csv')).sort_values('id')\ny_all = labels['label'].map(CLASS_TO_IDX).values.astype(np.int64)\n\ntest_dir = extract(find('test.7z'), WORK, 'test')\nX_test = cached('test_x.npy', lambda: load_folder(test_dir))\nprint('train', X_all.shape, 'test', X_test.shape, f'loaded in {time.time() - t0:.0f}s')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:07:37.425181Z","iopub.execute_input":"2026-10-10T08:07:37.425879Z","iopub.status.idle":"2026-10-10T08:09:20.435694Z","shell.execute_reply.started":"2026-10-10T08:07:37.425853Z","shell.execute_reply":"2026-10-10T08:09:20.434791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train / validation split\nidx = np.random.RandomState(CFG['seed']).permutation(len(X_all))\nval_idx, tr_idx = idx[:CFG['val_size']], idx[CFG['val_size']:]\n\ndef to_gpu_u8(x):  # NHWC uint8 numpy -> NCHW uint8 tensor on GPU\n    return torch.from_numpy(x).permute(0, 3, 1, 2).contiguous().to(dev)\n\nXtr, Ytr = to_gpu_u8(X_all[tr_idx]), torch.from_numpy(y_all[tr_idx]).to(dev)\nXva, Yva = (to_gpu_u8(X_all[val_idx]), torch.from_numpy(y_all[val_idx]).to(dev)) if len(val_idx) else (None, None)\nprint('train', tuple(Xtr.shape), 'val', None if Xva is None else tuple(Xva.shape))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:09:33.05847Z","iopub.execute_input":"2026-10-10T08:09:33.058909Z","iopub.status.idle":"2026-10-10T08:09:33.531326Z","shell.execute_reply.started":"2026-10-10T08:09:33.058882Z","shell.execute_reply":"2026-10-10T08:09:33.530599Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. GPU data augmentation: random crop + flip + CutMix","metadata":{}},{"cell_type":"code","source":"MEAN = torch.tensor([0.4914, 0.4822, 0.4465], device=dev).view(1, 3, 1, 1) * 255\nSTD = torch.tensor([0.2470, 0.2435, 0.2616], device=dev).view(1, 3, 1, 1) * 255\n\ndef normalize(x_u8):\n    return (x_u8.float() - MEAN.to(x_u8.device)) / STD.to(x_u8.device)\n\ndef random_crop_flip(x, pad=4):\n    \"\"\"Per-image random 32x32 crop from a reflect-padded 40x40 image, plus random horizontal flip.\"\"\"\n    B, C, H, W = x.shape\n    xp = F.pad(x, (pad, pad, pad, pad), mode='reflect')\n    i = torch.randint(0, 2 * pad + 1, (B,), device=x.device)\n    j = torch.randint(0, 2 * pad + 1, (B,), device=x.device)\n    rows = i[:, None] + torch.arange(H, device=x.device)          # (B, H)\n    cols = j[:, None] + torch.arange(W, device=x.device)          # (B, W)\n    b = torch.arange(B, device=x.device)[:, None, None, None]\n    c = torch.arange(C, device=x.device)[None, :, None, None]\n    x = xp[b, c, rows[:, None, :, None], cols[:, None, None, :]]\n    flip = torch.rand(B, device=x.device) < 0.5\n    return torch.where(flip[:, None, None, None], x.flip(3), x)\n\ndef cutmix(x, y, alpha=1.0):\n    \"\"\"CutMix (Yun et al., 2019): paste a random box from a shuffled copy of the batch.\"\"\"\n    B, _, H, W = x.shape\n    lam = np.random.beta(alpha, alpha)\n    perm = torch.randperm(B, device=x.device)\n    r = math.sqrt(1.0 - lam)\n    ch, cw = int(H * r), int(W * r)\n    cy, cx = np.random.randint(H), np.random.randint(W)\n    y1, y2 = np.clip(cy - ch // 2, 0, H), np.clip(cy + ch // 2, 0, H)\n    x1, x2 = np.clip(cx - cw // 2, 0, W), np.clip(cx + cw // 2, 0, W)\n    x[:, :, y1:y2, x1:x2] = x[perm, :, y1:y2, x1:x2]\n    lam = 1.0 - (y2 - y1) * (x2 - x1) / (H * W)   # exact area ratio after clipping\n    return x, y, y[perm], lam","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:09:37.592376Z","iopub.execute_input":"2026-10-10T08:09:37.592653Z","iopub.status.idle":"2026-10-10T08:09:37.631945Z","shell.execute_reply.started":"2026-10-10T08:09:37.592632Z","shell.execute_reply":"2026-10-10T08:09:37.631402Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Custom network: ResNet-SE\n\n| stage | resolution | layers | channels |\n|---|---|---|---|\n| stem | 32×32 | conv3×3-BN-ReLU | 64 |\n| stage 1 | 32→16 | conv3×3-BN-ReLU, max-pool, 2 × SE-residual block | 128 |\n| stage 2 | 16→8 | conv3×3-BN-ReLU, max-pool, 1 × SE-residual block | 256 |\n| stage 3 | 8→4 | conv3×3-BN-ReLU, max-pool, 2 × SE-residual block | 512 |\n| head | 4×4 → 1 | concat(global avg-pool, global max-pool) → linear | 1024 → 10 |\n\nDownsampling uses conv + max-pool rather than strided convolutions, so most of the compute is spent\nat the low resolutions. That makes it about as fast as ResNet-18 at ~12.8M parameters.","metadata":{}},{"cell_type":"code","source":"def conv_bn_relu(cin, cout):\n    return nn.Sequential(nn.Conv2d(cin, cout, 3, padding=1, bias=False),\n                         nn.BatchNorm2d(cout), nn.ReLU(inplace=True))\n\nclass SqueezeExcite(nn.Module):\n    def __init__(self, c, r=8):\n        super().__init__()\n        self.fc1 = nn.Conv2d(c, c // r, 1)\n        self.fc2 = nn.Conv2d(c // r, c, 1)\n    def forward(self, x):\n        s = F.adaptive_avg_pool2d(x, 1)\n        s = torch.sigmoid(self.fc2(F.relu(self.fc1(s), inplace=True)))\n        return x * s\n\nclass SEResBlock(nn.Module):\n    def __init__(self, c):\n        super().__init__()\n        self.conv1 = conv_bn_relu(c, c)\n        self.conv2 = nn.Conv2d(c, c, 3, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(c)\n        nn.init.zeros_(self.bn2.weight)  # each block starts as identity, which stabilises early training\n        self.se = SqueezeExcite(c)\n    def forward(self, x):\n        out = self.se(self.bn2(self.conv2(self.conv1(x))))\n        return F.relu(x + out, inplace=True)\n\nclass ResNetSE(nn.Module):\n    def __init__(self, num_classes=10, widths=(64, 128, 256, 512), blocks=(2, 1, 2)):\n        super().__init__()\n        self.stem = conv_bn_relu(3, widths[0])\n        stages = []\n        for cin, cout, nb in zip(widths[:-1], widths[1:], blocks):\n            stages += [conv_bn_relu(cin, cout), nn.MaxPool2d(2)]\n            stages += [SEResBlock(cout) for _ in range(nb)]\n        self.stages = nn.Sequential(*stages)\n        self.fc = nn.Linear(2 * widths[-1], num_classes)\n        for m in self.modules():  # random initialisation (He / Kaiming)\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n    def forward(self, x):\n        x = self.stages(self.stem(x))\n        x = torch.cat([F.adaptive_avg_pool2d(x, 1), F.adaptive_max_pool2d(x, 1)], dim=1).flatten(1)\n        return self.fc(x)\n\nmodel = ResNetSE().to(dev).to(memory_format=torch.channels_last)\nprint(f'parameters: {sum(p.numel() for p in model.parameters()) / 1e6:.2f}M')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:09:46.182432Z","iopub.execute_input":"2026-10-10T08:09:46.182834Z","iopub.status.idle":"2026-10-10T08:09:46.42363Z","shell.execute_reply.started":"2026-10-10T08:09:46.182809Z","shell.execute_reply":"2026-10-10T08:09:46.422945Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. AdamW, warmup + cosine schedule, EMA","metadata":{}},{"cell_type":"code","source":"decay, no_decay = [], []\nfor n_, p in model.named_parameters():\n    (no_decay if p.ndim <= 1 else decay).append(p)   # no weight decay on BN weights / biases\noptimizer = torch.optim.AdamW([{'params': decay, 'weight_decay': CFG['weight_decay']},\n                               {'params': no_decay, 'weight_decay': 0.0}],\n                              lr=CFG['peak_lr'], betas=(0.9, 0.999))\n\nsteps_per_epoch = len(Xtr) // CFG['batch_size']\ntotal_steps = CFG['epochs'] * steps_per_epoch\nwarmup_steps = CFG['warmup_epochs'] * steps_per_epoch\n\ndef warmup_cosine(step):\n    if step < warmup_steps:                                   # linear warmup 0 -> peak\n        return (step + 1) / warmup_steps\n    progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)\n    m = CFG['min_lr_ratio']                                   # cosine annealing peak -> m * peak\n    return m + (1 - m) * 0.5 * (1 + math.cos(math.pi * progress))\n\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, warmup_cosine)\n\nclass EMA:\n    \"\"\"Exponential moving average of weights and BN statistics.\"\"\"\n    def __init__(self, model, decay):\n        self.decay, self.updates = decay, 0\n        self.module = copy.deepcopy(model).eval()\n        for p in self.module.parameters():\n            p.requires_grad_(False)\n    @torch.no_grad()\n    def update(self, model):\n        self.updates += 1\n        d = min(self.decay, (1 + self.updates) / (10 + self.updates))  # EMA warmup: tracks closely early on\n        for e, m in zip(self.module.state_dict().values(), model.state_dict().values()):\n            if e.dtype.is_floating_point:\n                e.lerp_(m, 1 - d)\n            else:\n                e.copy_(m)\n\nema = EMA(model, CFG['ema_decay'])\ncriterion = nn.CrossEntropyLoss(label_smoothing=CFG['label_smoothing'])\nuse_amp = dev.type == 'cuda'\nscaler = (torch.amp.GradScaler('cuda', enabled=use_amp) if hasattr(torch.amp, 'GradScaler')\n          else torch.cuda.amp.GradScaler(enabled=use_amp))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:09:50.569724Z","iopub.execute_input":"2026-10-10T08:09:50.570008Z","iopub.status.idle":"2026-10-10T08:09:56.815164Z","shell.execute_reply.started":"2026-10-10T08:09:50.569987Z","shell.execute_reply":"2026-10-10T08:09:56.814331Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Train","metadata":{}},{"cell_type":"code","source":"@torch.no_grad()\ndef predict_proba(net, X_u8, bs=1024, tta=True):\n    net.eval()\n    out = []\n    for i in range(0, len(X_u8), bs):\n        x = normalize(X_u8[i:i + bs].to(dev, non_blocking=True)).contiguous(memory_format=torch.channels_last)\n        with torch.autocast(dev.type, dtype=torch.float16, enabled=use_amp):\n            p = net(x).float().softmax(1)\n            if tta:\n                p += net(x.flip(3)).float().softmax(1)\n        out.append(p.cpu())\n    return torch.cat(out)\n\ndef accuracy(net, X, Y):\n    return (predict_proba(net, X).argmax(1) == Y.cpu()).float().mean().item()\n\ndef train_one_epoch(epoch):\n    model.train()\n    perm = torch.randperm(len(Xtr), device=dev)\n    allow_mix = epoch < CFG['epochs'] - CFG['no_mix_last_epochs']\n    total_loss = 0.0\n    for k in range(steps_per_epoch):\n        idx = perm[k * CFG['batch_size']:(k + 1) * CFG['batch_size']]\n        x, y = random_crop_flip(normalize(Xtr[idx])), Ytr[idx]\n        mixed = allow_mix and random.random() < CFG['cutmix_prob']\n        if mixed:\n            x, ya, yb, lam = cutmix(x, y, CFG['cutmix_alpha'])\n        x = x.contiguous(memory_format=torch.channels_last)\n        with torch.autocast(dev.type, dtype=torch.float16, enabled=use_amp):\n            logits = model(x)\n            loss = lam * criterion(logits, ya) + (1 - lam) * criterion(logits, yb) if mixed else criterion(logits, y)\n        optimizer.zero_grad(set_to_none=True)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()\n        ema.update(model)\n        total_loss += loss.item()\n    return total_loss / steps_per_epoch\n\nhistory = []\nt_start = time.time()\nfor epoch in range(CFG['epochs']):\n    t = time.time()\n    loss = train_one_epoch(epoch)\n    row = dict(epoch=epoch + 1, loss=loss, lr=optimizer.param_groups[0]['lr'], sec=time.time() - t)\n    if Xva is not None and ((epoch + 1) % 5 == 0 or epoch + 1 == CFG['epochs']):\n        row['val_acc'] = accuracy(model, Xva, Yva)\n        row['val_acc_ema'] = accuracy(ema.module, Xva, Yva)\n    history.append(row)\n    print(' '.join(f'{k}={v:.4f}' if isinstance(v, float) else f'{k}={v}' for k, v in row.items()))\nprint(f'total training time: {(time.time() - t_start) / 60:.1f} min')\npd.DataFrame(history).to_csv('history.csv', index=False)\n\n# Use the EMA weights unless the validation split says the raw model is better\nfinal_model = ema.module\nif Xva is not None and history[-1]['val_acc'] > history[-1]['val_acc_ema']:\n    final_model = model\nprint('final model:', 'EMA' if final_model is ema.module else 'raw',\n      '| val acc:', None if Xva is None else max(history[-1]['val_acc'], history[-1]['val_acc_ema']))\ntorch.save(final_model.state_dict(), 'resnet_se_final.pt')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:10:01.417221Z","iopub.execute_input":"2026-10-10T08:10:01.418006Z","iopub.status.idle":"2026-10-10T08:41:06.877073Z","shell.execute_reply.started":"2026-10-10T08:10:01.417978Z","shell.execute_reply":"2026-10-10T08:41:06.876405Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Curves for the report","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nh = pd.DataFrame(history)\nfig, ax = plt.subplots(1, 3, figsize=(16, 4))\nax[0].plot(h.epoch, h.loss); ax[0].set_title('training loss'); ax[0].set_xlabel('epoch')\nax[1].plot(h.epoch, h.lr); ax[1].set_title('learning rate (warmup + cosine)'); ax[1].set_xlabel('epoch')\nif 'val_acc' in h:\n    v = h.dropna(subset=['val_acc'])\n    ax[2].plot(v.epoch, v.val_acc, label='model'); ax[2].plot(v.epoch, v.val_acc_ema, label='EMA')\n    ax[2].set_title('validation accuracy (flip TTA)'); ax[2].legend(); ax[2].set_xlabel('epoch')\nplt.tight_layout(); plt.savefig('curves.png', dpi=120); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T08:41:06.878415Z","iopub.execute_input":"2026-10-10T08:41:06.87863Z","iopub.status.idle":"2026-10-10T08:41:07.627791Z","shell.execute_reply.started":"2026-10-10T08:41:06.878613Z","shell.execute_reply":"2026-10-10T08:41:07.627074Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Predict the 300,000 test images → submission.csv","metadata":{}},{"cell_type":"code","source":"X_test_t = torch.from_numpy(X_test).permute(0, 3, 1, 2).contiguous()  # stays on CPU, batched to GPU\npreds = predict_proba(final_model, X_test_t, bs=2048).argmax(1).numpy()\nsub = pd.DataFrame({'id': np.arange(1, len(preds) + 1), 'label': [CLASSES[p] for p in preds]})\nsub.to_csv('submission.csv', index=False)\nprint(sub.head(), sub.shape)\nprint(sub.label.value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T09:13:51.353859Z","iopub.execute_input":"2026-10-10T09:13:51.354656Z","iopub.status.idle":"2026-10-10T09:14:49.948563Z","shell.execute_reply.started":"2026-10-10T09:13:51.354628Z","shell.execute_reply":"2026-10-10T09:14:49.947824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}