{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =========================\n# Cell 1: Config & Setup\n# =========================\nimport os, io, re, time, math, glob, random, shutil, copy\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.amp import GradScaler, autocast          # AMP API\nimport torchvision\nimport torchvision.transforms as T\n# REMOVED: from torchvision.models import resnet18\nfrom torchvision.transforms import RandAugment, RandomErasing\n\n@dataclass\nclass Cfg:\n    work_dir: str = \"./work\"\n    seed: int = 1337\n    device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n    # Data\n    img_size: int = 32\n    batch_size: int = 256\n    num_workers: int = 4\n\n    # Train\n    epochs: int = 150\n    base_lr: float = 0.20\n    weight_decay: float = 5e-4\n    momentum: float = 0.9\n    nesterov: bool = True\n\n    # Aug & regularization\n    use_mixup: bool = True\n    use_cutmix: bool = True\n    mix_alpha: float = 0.2\n    cut_alpha: float = 1.0\n\n    # EMA & AMP\n    use_ema: bool = True\n    ema_decay: float = 0.999\n    use_amp: bool = True\n\n    # Submission\n    submit_stream: bool = True\n    tta_hflip: bool = True\n    submit_batch: int = 1024\n\ncfg = Cfg()\nos.makedirs(cfg.work_dir, exist_ok=True)\n\n# Repro & device\ndef set_seed(seed=1337):\n    random.seed(seed); np.random.seed(seed)\n    torch.manual_seed(seed); torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = False\n    torch.backends.cudnn.benchmark = True\n\nset_seed(cfg.seed)\ndevice = torch.device(cfg.device)\nprint(\"Device:\", device)\n\n# CIFAR-10 normalization & classes\nCIFAR_MEAN = (0.4914, 0.4822, 0.4465)\nCIFAR_STD  = (0.2470, 0.2435, 0.2616)\nCLASS_NAMES = [\"airplane\",\"automobile\",\"bird\",\"cat\",\"deer\",\"dog\",\"frog\",\"horse\",\"ship\",\"truck\"]\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-01T13:08:28.770309Z","iopub.execute_input":"2025-10-01T13:08:28.771000Z","iopub.status.idle":"2025-10-01T13:08:39.623938Z","shell.execute_reply.started":"2025-10-01T13:08:28.770976Z","shell.execute_reply":"2025-10-01T13:08:39.623208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Cell 2: Data & Dataloaders\n# =========================\n# Strong but cheap augmentations\ntrain_tfms = T.Compose([\n    T.RandomCrop(cfg.img_size, padding=4),\n    T.RandomHorizontalFlip(),\n    RandAugment(num_ops=2, magnitude=9),\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n    RandomErasing(p=0.25, scale=(0.02, 0.2), ratio=(0.3, 3.3), value='random'),\n])\ntest_tfms  = T.Compose([\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\ntrain_set = torchvision.datasets.CIFAR10(root=\"./data\", train=True,  download=True, transform=train_tfms)\ntest_set  = torchvision.datasets.CIFAR10(root=\"./data\", train=False, download=True, transform=test_tfms)\n\ndef to_onehot(y, num_classes=10):\n    return F.one_hot(y, num_classes=num_classes).float()\n\ndef mixup_cutmix_collate(batch):\n    images, labels = zip(*batch)\n    x = torch.stack(images, dim=0)\n    y = torch.tensor(labels, dtype=torch.long)\n    y_oh = to_onehot(y, 10)\n\n    if cfg.use_cutmix and random.random() < 0.5:\n        lam = np.random.beta(cfg.cut_alpha, cfg.cut_alpha)\n        B, C, H, W = x.size()\n        idx = torch.randperm(B)\n        cut_w = int(W * math.sqrt(1 - lam))\n        cut_h = int(H * math.sqrt(1 - lam))\n        cx = np.random.randint(W); cy = np.random.randint(H)\n        x1 = np.clip(cx - cut_w // 2, 0, W); x2 = np.clip(cx + cut_w // 2, 0, W)\n        y1 = np.clip(cy - cut_h // 2, 0, H); y2 = np.clip(cy + cut_h // 2, 0, H)\n        x[:, :, y1:y2, x1:x2] = x[idx, :, y1:y2, x1:x2]\n        lam_adj = 1 - ((x2 - x1) * (y2 - y1)) / (W * H)\n        y_oh = lam_adj * y_oh + (1 - lam_adj) * y_oh[idx]\n    elif cfg.use_mixup:\n        lam = np.random.beta(cfg.mix_alpha, cfg.mix_alpha)\n        B = x.size(0); idx = torch.randperm(B)\n        x = lam * x + (1 - lam) * x[idx]\n        y_oh = lam * y_oh + (1 - lam) * y_oh[idx]\n\n    return x, y_oh\n\ntrain_loader = DataLoader(\n    train_set, batch_size=cfg.batch_size, shuffle=True, drop_last=True,\n    num_workers=cfg.num_workers, pin_memory=True,\n    collate_fn=mixup_cutmix_collate if (cfg.use_mixup or cfg.use_cutmix) else None,\n)\nval_loader = DataLoader(\n    test_set, batch_size=1024, shuffle=False,\n    num_workers=cfg.num_workers, pin_memory=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-01T13:08:42.541882Z","iopub.execute_input":"2025-10-01T13:08:42.542682Z","iopub.status.idle":"2025-10-01T13:08:51.166080Z","shell.execute_reply.started":"2025-10-01T13:08:42.542660Z","shell.execute_reply":"2025-10-01T13:08:51.165468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# =========================\n# Cell 3: Model, Optimizer, LR, EMA, AMP\n# =========================\n# ---- UPDATED: custom residual network (basic layers only) ----\nclass BasicBlock(nn.Module):\n    expansion = 1\n    def __init__(self, in_ch, out_ch, stride=1):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_ch, out_ch, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1   = nn.BatchNorm2d(out_ch)\n        self.relu  = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(out_ch, out_ch, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2   = nn.BatchNorm2d(out_ch)\n\n        self.proj = None\n        if stride != 1 or in_ch != out_ch:\n            self.proj = nn.Sequential(\n                nn.Conv2d(in_ch, out_ch, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_ch)\n            )\n\n        # He init\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.ones_(m.weight); nn.init.zeros_(m.bias)\n        # Zero-init last BN on residual branch\n        nn.init.zeros_(self.bn2.weight)\n\n    def forward(self, x):\n        identity = x\n        out = self.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        if self.proj is not None:\n            identity = self.proj(identity)\n        out = self.relu(out + identity)\n        return out\n\ndef _make_layer(in_ch, out_ch, num_blocks, first_stride):\n    layers = [BasicBlock(in_ch, out_ch, stride=first_stride)]\n    for _ in range(num_blocks - 1):\n        layers.append(BasicBlock(out_ch, out_ch, stride=1))\n    return nn.Sequential(*layers)\n\nclass SmallResNet(nn.Module):\n    \"\"\"\n    CIFAR-10 residual net (all basic 3x3 blocks):\n      Stem: 3x3, 64\n      Stages: [64]x2, [128]x2, [256]x2, [512]x2 with first block stride=2 except stage1\n      Head: global avg pool -> FC(512, 10)\n    \"\"\"\n    def __init__(self, num_classes=10):\n        super().__init__()\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n        )\n        self.layer1 = _make_layer(64,   64,  num_blocks=2, first_stride=1)  # 32x32\n        self.layer2 = _make_layer(64,   128, num_blocks=2, first_stride=2)  # 16x16\n        self.layer3 = _make_layer(128,  256, num_blocks=2, first_stride=2)  # 8x8\n        self.layer4 = _make_layer(256,  512, num_blocks=2, first_stride=2)  # 4x4\n        self.pool   = nn.AdaptiveAvgPool2d((1,1))\n        self.fc     = nn.Linear(512, num_classes)\n\n        nn.init.kaiming_normal_(self.fc.weight, mode='fan_out', nonlinearity='relu')\n        nn.init.zeros_(self.fc.bias)\n\n    def forward(self, x):\n        x = self.stem(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        x = self.pool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n        return x\n\n# Model — custom residual (no pretrained/backbone imports)\nmodel = SmallResNet(num_classes=10).to(device)\n\n# Weight decay: apply to weights only (not BN/bias)\ndef split_weights(m):\n    decay, no_decay = [], []\n    for name, p in m.named_parameters():\n        if not p.requires_grad:\n            continue\n        if p.ndim == 1 or name.endswith(\".bias\"):  # BN/bias\n            no_decay.append(p)\n        else:\n            decay.append(p)\n    return [\n        {\"params\": decay,    \"weight_decay\": cfg.weight_decay},\n        {\"params\": no_decay, \"weight_decay\": 0.0},\n    ]\n\noptimizer = torch.optim.SGD(\n    split_weights(model), lr=cfg.base_lr,\n    momentum=cfg.momentum, nesterov=cfg.nesterov\n)\n\n# 5-epoch warmup + cosine\ndef cosine_with_warmup_lambda(epoch):\n    warmup = 5\n    if epoch < warmup:\n        return float(epoch + 1) / warmup\n    progress = (epoch - warmup) / max(1, (cfg.epochs - warmup))\n    return 0.5 * (1.0 + math.cos(math.pi * progress))\n\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=cosine_with_warmup_lambda)\n\n# Loss (supports soft labels)\ndef soft_ce_loss(logits, targets):\n    log_probs = F.log_softmax(logits, dim=1)\n    return -(targets * log_probs).sum(dim=1).mean()\n\n# AMP (new API)\nscaler = GradScaler('cuda', enabled=cfg.use_amp)\n\n# EMA (safe deepcopy-based)\nclass ModelEMA:\n    def __init__(self, model, decay=cfg.ema_decay):\n        self.decay = decay\n        self.ema = copy.deepcopy(model).to(next(model.parameters()).device)\n        self.ema.eval()\n        for p in self.ema.parameters():\n            p.requires_grad_(False)\n        self._backup = None\n\n    @torch.no_grad()\n    def update(self, model):\n        msd = model.state_dict()\n        esd = self.ema.state_dict()\n        for k in esd.keys():\n            if esd[k].dtype.is_floating_point:\n                esd[k].mul_(self.decay).add_(msd[k].detach(), alpha=1.0 - self.decay)\n            else:\n                esd[k].copy_(msd[k])\n\n    def store(self, model):\n        self._backup = copy.deepcopy(model.state_dict())\n\n    def copy_to(self, model):\n        model.load_state_dict(self.ema.state_dict(), strict=True)\n\n    def restore(self, model):\n        if self._backup is not None:\n            model.load_state_dict(self._backup, strict=True)\n            self._backup = None\n\nema = ModelEMA(model) if cfg.use_ema else None\nce = nn.CrossEntropyLoss()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-01T13:09:02.234627Z","iopub.execute_input":"2025-10-01T13:09:02.235397Z","iopub.status.idle":"2025-10-01T13:09:02.422180Z","shell.execute_reply.started":"2025-10-01T13:09:02.235370Z","shell.execute_reply":"2025-10-01T13:09:02.421384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# =========================\n# Cell 4: Train / Eval / Loop\n# =========================\ndef _train_acc_from_logits(logits, targets):\n    preds = logits.argmax(1)\n    tgt = targets.argmax(1) if targets.ndim == 2 else targets\n    return (preds == tgt).float().mean().item()\n\ndef train_one_epoch(epoch):\n    model.train()\n    running_loss, running_acc, n = 0.0, 0.0, 0\n    t0 = time.time()\n    for images, targets in train_loader:\n        images = images.to(device, non_blocking=True)\n        targets = targets.to(device, non_blocking=True)\n        optimizer.zero_grad(set_to_none=True)\n\n        with autocast(device_type='cuda', enabled=cfg.use_amp):\n            logits = model(images)\n            loss = soft_ce_loss(logits, targets) if targets.ndim == 2 else ce(logits, targets)\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        if ema: ema.update(model)\n\n        bs = images.size(0)\n        running_loss += loss.item() * bs\n        running_acc  += _train_acc_from_logits(logits.detach(), targets) * bs\n        n += bs\n\n    scheduler.step()\n    return running_loss / n, running_acc / n, time.time() - t0\n\n@torch.no_grad()\ndef evaluate(use_ema=True):\n    if use_ema and ema:\n        ema.store(model); ema.copy_to(model)\n    model.eval()\n    total_loss, total_acc, n = 0.0, 0.0, 0\n    for images, labels in val_loader:\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        logits = model(images)\n        loss = ce(logits, labels)\n        acc = (logits.argmax(1) == labels).float().mean().item()\n        bs = images.size(0)\n        total_loss += loss.item() * bs\n        total_acc  += acc * bs\n        n += bs\n    if use_ema and ema:\n        ema.restore(model)\n    return total_loss / n, total_acc / n\n\n# Training loop\nbest_acc = 0.0\nbest_path = os.path.join(cfg.work_dir, \"best_model.pth\")\n\nfor epoch in range(cfg.epochs):\n    tr_loss, tr_acc, dt = train_one_epoch(epoch)\n    val_loss, val_acc = evaluate(use_ema=True)\n\n    if val_acc > best_acc:\n        best_acc = val_acc\n        torch.save((ema.ema if (cfg.use_ema and ema) else model).state_dict(), best_path)\n\n    lr_now = optimizer.param_groups[0]['lr']\n    print(f\"Epoch {epoch+1:03d}/{cfg.epochs} | lr {lr_now:.5f} | \"\n          f\"train_loss {tr_loss:.4f} acc {tr_acc:.4f} | \"\n          f\"val_loss {val_loss:.4f} acc {val_acc:.4f} | {dt:.1f}s\")\n\nprint(\"Best Val Acc:\", best_acc)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-01T13:09:05.817974Z","iopub.execute_input":"2025-10-01T13:09:05.818809Z","iopub.status.idle":"2025-10-01T14:02:42.118607Z","shell.execute_reply.started":"2025-10-01T13:09:05.818780Z","shell.execute_reply":"2025-10-01T14:02:42.117549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Cell 5: Offline test accuracy (same split as val_loader)\n# =========================\nassert os.path.exists(best_path), f\"Checkpoint not found: {best_path}\"\ntest_loader_off = DataLoader(test_set, batch_size=1024, shuffle=False,\n                             num_workers=cfg.num_workers, pin_memory=True)\n\neval_model = SmallResNet(num_classes=10).to(device).eval()\nstate = torch.load(best_path, map_location=device)\neval_model.load_state_dict(state, strict=True)\neval_model.eval()\n\ncorrect, total = 0, 0\nwith torch.inference_mode():\n    for xb, yb in test_loader_off:\n        xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n        pred = eval_model(xb).argmax(1)\n        correct += (pred == yb).sum().item()\n        total   += yb.size(0)\nfinal_acc = correct / total\nprint(f\"[OFFLINE] CIFAR-10 test accuracy: {final_acc:.4f} ({final_acc*100:.2f}%)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-01T15:00:30.297227Z","iopub.execute_input":"2025-10-01T15:00:30.297864Z","iopub.status.idle":"2025-10-01T15:00:32.204244Z","shell.execute_reply.started":"2025-10-01T15:00:30.297841Z","shell.execute_reply":"2025-10-01T15:00:32.203386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Cell 6  Extract \n# =========================\n!pip -q install py7zr\nimport py7zr\n\n# Find archive\ncandidates = glob.glob('/kaggle/input/**/test.7z', recursive=True)\narchive_path = candidates[0]\n\nshutil.rmtree('test', ignore_errors=True)\nos.makedirs('test', exist_ok=True)\nwith py7zr.SevenZipFile(archive_path, mode='r') as z:\n    z.extractall(path='test')\n\npng_files = glob.glob('test/**/*.png', recursive=True)\nprint(\"PNG files:\", len(png_files))\nassert len(png_files) == 300000, f\"Expected 300000, got {len(png_files)}\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-01T15:09:34.187140Z","iopub.execute_input":"2025-10-01T15:09:34.187837Z","iopub.status.idle":"2025-10-01T15:12:11.782750Z","shell.execute_reply.started":"2025-10-01T15:09:34.187807Z","shell.execute_reply":"2025-10-01T15:12:11.781848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Cell 7: Submit (Kaggle-only)\n# =========================\nimport os, re, glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nimport torch\n\n# 0) Ensure test/ exists and has 300k PNGs (run Cell 6 first)\npng_count = len(glob.glob('test/**/*.png', recursive=True))\nif png_count != 300000:\n    raise RuntimeError(\n        f\"'test' folder not ready: found {png_count} PNGs. \"\n        \"Run Cell 6 to extract the Kaggle test set, then rerun this cell.\"\n    )\n\n# 1) Transforms\nval_tfms = T.Compose([\n    T.Resize((cfg.img_size, cfg.img_size), antialias=True),\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\n# 2) Dataset (sorted by numeric id)\nclass KaggleTestDataset(Dataset):\n    def __init__(self, root=\"test\", transform=None):\n        self.transform = transform\n        files = glob.glob(os.path.join(root, \"**\", \"*.png\"), recursive=True)\n\n        def keyfn(p):\n            b = os.path.basename(p)\n            m = re.match(r\"(\\d+)\", os.path.splitext(b)[0])\n            return int(m.group(1)) if m else 10**12\n\n        files.sort(key=keyfn)\n        self.paths = files\n        self.ids = [int(os.path.splitext(os.path.basename(p))[0]) for p in self.paths]\n\n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, i):\n        x = self.transform(Image.open(self.paths[i]).convert(\"RGB\"))\n        return x, self.ids[i]\n\ntest_ds = KaggleTestDataset(transform=val_tfms)\ntest_loader_submit = DataLoader(\n    test_ds,\n    batch_size=cfg.submit_batch,\n    shuffle=False,\n    num_workers=cfg.num_workers,\n    pin_memory=True\n)\n\n# 3) Load best checkpoint with the SAME model used for training\nassert os.path.exists(best_path), f\"Checkpoint not found: {best_path}. Train first.\"\nsubmit_model = SmallResNet(num_classes=10).to(device).eval()\nsubmit_model.load_state_dict(torch.load(best_path, map_location=device), strict=True)\nsubmit_model.eval()\n\n# 4) Inference (+ optional TTA)\nall_ids, all_preds = [], []\nwith torch.inference_mode():\n    for xb, idb in test_loader_submit:\n        xb = xb.to(device, non_blocking=True)\n        logits = submit_model(xb)\n        if cfg.tta_hflip:\n            logits = (logits + submit_model(torch.flip(xb, dims=[3]))) / 2.0\n        preds = logits.argmax(1).cpu().numpy()\n        all_ids.append(idb.numpy())\n        all_preds.append(preds)\n\nids = np.concatenate(all_ids)\nlabels_idx = np.concatenate(all_preds)\nlabels = [CLASS_NAMES[i] for i in labels_idx]\n\n# 5) Build submission and sanity-check for Kaggle CIFAR-10\nsub = pd.DataFrame({\"id\": ids, \"label\": labels}).sort_values(\"id\")\nassert sub.shape[0] == 300000, f\"Expected 300000 rows, got {sub.shape[0]}\"\nassert sub[\"id\"].min() == 1 and sub[\"id\"].max() == 300000, \"IDs must span 1..300000\"\nassert sub[\"id\"].nunique() == 300000, \"IDs must be unique\"\n\n# 6) Save\nout_csv = os.path.join(cfg.work_dir, \"submission.csv\")\nos.makedirs(cfg.work_dir, exist_ok=True)\nsub.to_csv(out_csv, index=False)\nprint(sub.head(), \"\\nWrote:\", out_csv)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-01T15:24:43.641800Z","iopub.execute_input":"2025-10-01T15:24:43.642632Z","iopub.status.idle":"2025-10-01T15:25:53.239217Z","shell.execute_reply.started":"2025-10-01T15:24:43.642603Z","shell.execute_reply":"2025-10-01T15:25:53.238094Z"}},"outputs":[],"execution_count":null}]}