{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =============================================================================\n# CIFAR-10 — CUSTOM ConvNet (not an off-the-shelf architecture), trained\n# from scratch, satisfying ALL of:\n#   (1) randomly initialized weights, no pre-trained weights\n#   (2) advanced optimizer: AdamW\n#   (3) cosine annealing LR schedule WITH a linear warmup phase\n#   (4) advanced data augmentation: CutMix (on top of standard crop/flip)\n#\n# Paste into Kaggle Notebook cells split on \"# %%\", or run top-to-bottom\n# in Colab. Expect ~45-70 min for 80 epochs on a single T4/P100.\n# =============================================================================\n\n# %% [1] Setup --------------------------------------------------------------\nimport os, time, glob, re, math\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nimport torchvision\nimport torchvision.transforms as T\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint('Using device:', device)\ntorch.manual_seed(42)\nnp.random.seed(42)\n\nCLASSES = None  # filled in after loading the dataset (alphabetical order)\n\n# %% [2] Data -----------------------------------------------------------------\nstats = ((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))\n\n# NOTE: CutMix is applied on raw batches inside the training loop (it needs\n# pairs of images), so the usual per-image \"Cutout\" transform is NOT used\n# here — CutMix is the advanced augmentation satisfying that requirement.\n# Standard crop/flip are still used underneath (not a substitute for it).\ntrain_tfms = T.Compose([\n    T.RandomCrop(32, padding=4, padding_mode='reflect'),\n    T.RandomHorizontalFlip(),\n    T.ToTensor(),\n    T.Normalize(*stats),\n])\ntest_tfms = T.Compose([\n    T.ToTensor(),\n    T.Normalize(*stats),\n])\n\ntrain_ds = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=train_tfms)\ntest_ds  = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=test_tfms)\nCLASSES = train_ds.classes\nprint(CLASSES)\n\nBATCH = 256\ntrain_dl = DataLoader(train_ds, batch_size=BATCH, shuffle=True, num_workers=2,\n                       pin_memory=True, drop_last=True)\ntest_dl  = DataLoader(test_ds, batch_size=512, shuffle=False, num_workers=2, pin_memory=True)\n\n# %% [3] CutMix ----------------------------------------------------------------\ndef rand_bbox(size, lam):\n    \"\"\"Random bounding box for CutMix, covering a (1-lam) fraction of area.\"\"\"\n    H, W = size[2], size[3]\n    cut_rat = math.sqrt(1. - lam)\n    cut_h, cut_w = int(H * cut_rat), int(W * cut_rat)\n    cy, cx = np.random.randint(H), np.random.randint(W)\n    y1 = np.clip(cy - cut_h // 2, 0, H)\n    y2 = np.clip(cy + cut_h // 2, 0, H)\n    x1 = np.clip(cx - cut_w // 2, 0, W)\n    x2 = np.clip(cx + cut_w // 2, 0, W)\n    return x1, y1, x2, y2\n\ndef cutmix(x, y, alpha=1.0):\n    \"\"\"Mixes a batch with itself (shuffled) by pasting a random box from one\n    image onto another, and mixes the labels by the resulting area ratio.\"\"\"\n    lam = np.random.beta(alpha, alpha)\n    perm = torch.randperm(x.size(0), device=x.device)\n    x2, y2 = x[perm], y[perm]\n    x1b, y1b, x2b, y2b = rand_bbox(x.size(), lam)\n    x[:, :, y1b:y2b, x1b:x2b] = x2[:, :, y1b:y2b, x1b:x2b]\n    lam_adj = 1 - ((x2b - x1b) * (y2b - y1b) / (x.size(-1) * x.size(-2)))\n    return x, y, y2, lam_adj\n\n# %% [4] Model: a custom-designed CNN (NOT an off-the-shelf architecture) -----\n# Design choices (all mine, not copied from a published network):\n#   - \"Wide stem\": two 3x3 convs before any downsampling, to build low-level\n#     features at full resolution before losing spatial detail.\n#   - Each stage downsamples via a STRIDED conv (not pooling) and uses a\n#     custom \"SEResidualUnit\": two 3x3 convs with a residual connection,\n#     followed by a squeeze-and-excitation (channel attention) gate that\n#     reweights channels based on global context.\n#   - Channel widths (80 -> 160 -> 320 -> 640) are chosen off the standard\n#     64/128/256/512 grid to make this an intentionally distinct design.\n#   - Head: global average pool -> dropout -> linear.\n\nclass SqueezeExcite(nn.Module):\n    def __init__(self, ch, reduction=8):\n        super().__init__()\n        hidden = max(ch // reduction, 8)\n        self.fc1 = nn.Linear(ch, hidden)\n        self.fc2 = nn.Linear(hidden, ch)\n    def forward(self, x):\n        b, c, _, _ = x.shape\n        s = x.mean(dim=(2, 3))               # global average -> (b, c)\n        s = F.relu(self.fc1(s))\n        s = torch.sigmoid(self.fc2(s))\n        return x * s.view(b, c, 1, 1)\n\nclass SEResidualUnit(nn.Module):\n    def __init__(self, in_ch, out_ch, stride=1):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_ch, out_ch, 3, stride=stride, padding=1, bias=False)\n        self.bn1   = nn.BatchNorm2d(out_ch)\n        self.conv2 = nn.Conv2d(out_ch, out_ch, 3, stride=1, padding=1, bias=False)\n        self.bn2   = nn.BatchNorm2d(out_ch)\n        self.se    = SqueezeExcite(out_ch)\n        self.shortcut = None\n        if stride != 1 or in_ch != out_ch:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_ch, out_ch, 1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_ch),\n            )\n    def forward(self, x):\n        identity = x if self.shortcut is None else self.shortcut(x)\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        out = self.se(out)\n        return F.relu(out + identity)\n\nclass CustomCIFARNet(nn.Module):\n    def __init__(self, num_classes=10, widths=(80, 160, 320, 640), blocks_per_stage=2, dropout=0.2):\n        super().__init__()\n        w0, w1, w2, w3 = widths\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, w0, 3, padding=1, bias=False), nn.BatchNorm2d(w0), nn.ReLU(inplace=True),\n            nn.Conv2d(w0, w0, 3, padding=1, bias=False), nn.BatchNorm2d(w0), nn.ReLU(inplace=True),\n        )\n        self.stage1 = self._make_stage(w0, w1, blocks_per_stage, stride=1)   # 32x32\n        self.stage2 = self._make_stage(w1, w2, blocks_per_stage, stride=2)   # 16x16\n        self.stage3 = self._make_stage(w2, w3, blocks_per_stage, stride=2)   # 8x8\n        self.stage4 = self._make_stage(w3, w3, blocks_per_stage, stride=2)   # 4x4\n        self.pool = nn.AdaptiveAvgPool2d(1)\n        self.drop = nn.Dropout(dropout)\n        self.fc   = nn.Linear(w3, num_classes)\n\n    def _make_stage(self, in_ch, out_ch, n_blocks, stride):\n        layers = [SEResidualUnit(in_ch, out_ch, stride=stride)]\n        for _ in range(n_blocks - 1):\n            layers.append(SEResidualUnit(out_ch, out_ch, stride=1))\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.stem(x)\n        x = self.stage1(x)\n        x = self.stage2(x)\n        x = self.stage3(x)\n        x = self.stage4(x)\n        x = self.pool(x).flatten(1)\n        x = self.drop(x)\n        return self.fc(x)\n\nmodel = CustomCIFARNet().to(device)\nprint(f'{sum(p.numel() for p in model.parameters()):,} parameters (randomly initialized)')\n\n# %% [5] Optimizer: AdamW + cosine annealing WITH linear warmup --------------\nEPOCHS = 80\nWARMUP_EPOCHS = 5\nBASE_LR = 3e-3\nWEIGHT_DECAY = 0.05\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=BASE_LR, weight_decay=WEIGHT_DECAY)\n\nsteps_per_epoch = len(train_dl)\nwarmup_steps = WARMUP_EPOCHS * steps_per_epoch\ntotal_steps = EPOCHS * steps_per_epoch\n\ndef lr_lambda(step):\n    if step < warmup_steps:\n        return step / max(1, warmup_steps)                 # linear warmup 0 -> 1\n    progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)\n    return 0.5 * (1 + math.cos(math.pi * progress))          # cosine decay 1 -> 0\n\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1)\nscaler = torch.cuda.amp.GradScaler()\n\n# %% [6] Training loop (CutMix applied per-batch) -----------------------------\nCUTMIX_PROB = 0.5\nCUTMIX_ALPHA = 1.0\n\ndef evaluate(dl):\n    model.eval()\n    correct, total = 0, 0\n    with torch.no_grad():\n        for xb, yb in dl:\n            xb, yb = xb.to(device), yb.to(device)\n            with torch.cuda.amp.autocast():\n                out = model(xb)\n            correct += (out.argmax(1) == yb).sum().item()\n            total += yb.size(0)\n    return correct / total\n\nglobal_step = 0\nfor epoch in range(EPOCHS):\n    model.train()\n    t0 = time.time()\n    running_loss = 0.0\n    for xb, yb in train_dl:\n        xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n        optimizer.zero_grad(set_to_none=True)\n\n        use_cutmix = np.random.rand() < CUTMIX_PROB\n        with torch.cuda.amp.autocast():\n            if use_cutmix:\n                xb, y_a, y_b, lam = cutmix(xb, yb, alpha=CUTMIX_ALPHA)\n                out = model(xb)\n                loss = lam * criterion(out, y_a) + (1 - lam) * criterion(out, y_b)\n            else:\n                out = model(xb)\n                loss = criterion(out, yb)\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()\n        global_step += 1\n        running_loss += loss.item() * xb.size(0)\n\n    train_loss = running_loss / len(train_ds)\n    test_acc = evaluate(test_dl)\n    cur_lr = optimizer.param_groups[0]['lr']\n    print(f'Epoch {epoch+1:3d}/{EPOCHS} | loss {train_loss:.4f} | '\n          f'official test acc {test_acc:.4f} | lr {cur_lr:.5f} | {time.time()-t0:.1f}s')\n\nfinal_test_acc = evaluate(test_dl)\nprint(f'\\nFinal official-test accuracy (estimated Kaggle score): {final_test_acc:.4f}')\ntorch.save(model.state_dict(), 'customnet_cifar10.pth')\n\n# %% [7] Decompress the Kaggle test set (300,000 images) ----------------------\n!pip install -q py7zr\nimport py7zr, glob\nwith py7zr.SevenZipFile('/kaggle/input/competitions/cifar-10/test.7z', mode='r') as archive:\n    archive.extractall(path='.')\nprint('Extracted files:', len(glob.glob('test/*')))   # should be 300000\n\n# %% [8] Predict on Kaggle test set and write submission.csv ------------------\nfrom PIL import Image\nimport pandas as pd\n\nclass KaggleTestDataset(torch.utils.data.Dataset):\n    def __init__(self, folder, n_images, transform):\n        self.folder, self.n_images, self.transform = folder, n_images, transform\n    def __len__(self):\n        return self.n_images\n    def __getitem__(self, idx):\n        img_id = idx + 1\n        img = Image.open(os.path.join(self.folder, f'{img_id}.png')).convert('RGB')\n        return self.transform(img), img_id\n\nN_TEST = len(glob.glob('test/*'))\nkaggle_test_ds = KaggleTestDataset('test', N_TEST, test_tfms)\nkaggle_test_dl = DataLoader(kaggle_test_ds, batch_size=512, shuffle=False, num_workers=2)\n\nmodel.eval()\nall_ids, all_preds = [], []\nwith torch.no_grad():\n    for xb, ids in kaggle_test_dl:\n        xb = xb.to(device)\n        with torch.cuda.amp.autocast():\n            out = model(xb)\n        all_preds.extend(out.argmax(1).cpu().numpy().tolist())\n        all_ids.extend(ids.numpy().tolist())\n\nsubmission = pd.DataFrame({'id': all_ids, 'label': [CLASSES[p] for p in all_preds]})\nsubmission = submission.sort_values('id').reset_index(drop=True)\nsubmission.to_csv('submission.csv', index=False)\nprint(len(submission), submission['id'].min(), submission['id'].max())\nsubmission.head()\n\n# Then: Submit Predictions on Kaggle (or `kaggle competitions submit -c cifar-10 -f submission.csv -m \"CustomCIFARNet, AdamW, cosine+warmup, CutMix\"`)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-10T11:03:58.166106Z","iopub.execute_input":"2026-10-10T11:03:58.166324Z"}},"outputs":[],"execution_count":null}]}