{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# A) imports, seed, device, speed flags\nimport os, math, random, glob, json, time\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport torch, torch.nn as nn, torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED)\n\n# speed tweaks\nimport torch.backends.cudnn as cudnn\ncudnn.benchmark = True  # lets cudnn pick fastest conv algo for 32x32\n\nprint(\"Device:\", DEVICE)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-15T14:38:16.271785Z","iopub.execute_input":"2025-10-15T14:38:16.272036Z","iopub.status.idle":"2025-10-15T14:38:21.997567Z","shell.execute_reply.started":"2025-10-15T14:38:16.272018Z","shell.execute_reply":"2025-10-15T14:38:21.996844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# B) extract /kaggle/input/cifar-10/*.7z → /kaggle/working/{train,test}\n!apt -yq install libarchive-dev >/dev/null\n!pip -q install libarchive >/dev/null\nimport libarchive.public\n\nbase_in  = \"/kaggle/input/cifar-10\"\ntrain_out = \"/kaggle/working/train\"\ntest_out  = \"/kaggle/working/test\"\n\nos.makedirs(train_out, exist_ok=True)\nos.makedirs(test_out, exist_ok=True)\n\n# extract train once\nif len(glob.glob(train_out+\"/*.png\")) < 50000:\n    for _ in libarchive.public.file_pour(f\"{base_in}/train.7z\"): pass\n\n# extract test once (300,000 files; allow ~2–3 min)\nif len(glob.glob(test_out+\"/*.png\")) < 300000:\n    for _ in libarchive.public.file_pour(f\"{base_in}/test.7z\"): pass\n\nlabels_df = pd.read_csv(f\"{base_in}/trainLabels.csv\")  # columns: id, label\nclasses = sorted(labels_df.label.unique().tolist())    # list of class names\ncls_to_idx = {c:i for i,c in enumerate(classes)}\n\n# stratified split 45k/5k\nfrom sklearn.model_selection import StratifiedShuffleSplit\nsss = StratifiedShuffleSplit(n_splits=1, test_size=5000, random_state=SEED)\nidx_train, idx_val = next(sss.split(labels_df[\"id\"], labels_df[\"label\"]))\nlabels_tr = labels_df.iloc[idx_train].reset_index(drop=True)\nlabels_va = labels_df.iloc[idx_val].reset_index(drop=True)\n\n# transforms (CutMix happens in the loop; these are base TFMs)\nMEAN = (0.4914, 0.4822, 0.4465)\nSTD  = (0.2023, 0.1994, 0.2010)\n\ntfm_train = T.Compose([\n    T.RandomCrop(32, padding=4),\n    T.RandomHorizontalFlip(),\n    T.ColorJitter(0.2,0.2,0.2,0.1),\n    T.ToTensor(),\n    T.Normalize(MEAN, STD),\n    T.RandomErasing(p=0.25, scale=(0.02,0.2), ratio=(0.3,3.3), value=0)\n])\n\ntfm_test = T.Compose([\n    T.ToTensor(),\n    T.Normalize(MEAN, STD),\n])\n\nclass CifarFiles(Dataset):\n    def __init__(self, df, root, transform):\n        self.df = df\n        self.root = Path(root)\n        self.tfm = transform\n        self.has_label = \"label\" in df.columns\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, i):\n        row = self.df.iloc[i]\n        img = Image.open(self.root / f\"{row['id']}.png\").convert(\"RGB\")\n        x = self.tfm(img)\n        if self.has_label:\n            y = cls_to_idx[row[\"label\"]]\n            return x, torch.tensor(y, dtype=torch.long)\n        else:\n            return x, int(row[\"id\"])\n\ntrain_ds = CifarFiles(labels_tr, train_out, tfm_train)\nval_ds   = CifarFiles(labels_va, train_out, tfm_test)\n\nBS = 128\ntrain_loader = DataLoader(train_ds, batch_size=BS, shuffle=True, num_workers=2, pin_memory=True)\nval_loader   = DataLoader(val_ds,   batch_size=BS, shuffle=False, num_workers=2, pin_memory=True)\n\nprint(\"Train/Val sizes:\", len(train_ds), len(val_ds), \"Classes:\", classes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T14:38:21.998747Z","iopub.execute_input":"2025-10-15T14:38:21.999078Z","iopub.status.idle":"2025-10-15T14:39:50.936626Z","shell.execute_reply.started":"2025-10-15T14:38:21.999060Z","shell.execute_reply":"2025-10-15T14:39:50.935770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# C) model: scratch ResNet-ish for CIFAR-10 (random init)\ndef conv3(in_c, out_c, s=1): return nn.Conv2d(in_c, out_c, 3, stride=s, padding=1, bias=False)\n\nclass BasicBlock(nn.Module):\n    def __init__(self, in_c, out_c, stride=1):\n        super().__init__()\n        self.conv1 = conv3(in_c, out_c, stride)\n        self.bn1   = nn.BatchNorm2d(out_c)\n        self.conv2 = conv3(out_c, out_c, 1)\n        self.bn2   = nn.BatchNorm2d(out_c)\n        self.down  = None\n        if stride != 1 or in_c != out_c:\n            self.down = nn.Sequential(\n                nn.Conv2d(in_c, out_c, 1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_c)\n            )\n        self.act = nn.ReLU(inplace=True)\n\n    def forward(self, x):\n        identity = x\n        out = self.act(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        if self.down is not None:\n            identity = self.down(identity)\n        out = self.act(out + identity)\n        return out\n\nclass SmallResNetCIFAR(nn.Module):\n    def __init__(self, num_classes=10):\n        super().__init__()\n        self.stem = nn.Sequential(conv3(3,64), nn.BatchNorm2d(64), nn.ReLU(inplace=True))\n        # 32x32\n        self.s1 = nn.Sequential(BasicBlock(64,64,1), BasicBlock(64,64,1))\n        # 16x16\n        self.s2 = nn.Sequential(BasicBlock(64,128,2), BasicBlock(128,128,1))\n        # 8x8\n        self.s3 = nn.Sequential(BasicBlock(128,256,2), BasicBlock(256,256,1))\n        self.pool = nn.AdaptiveAvgPool2d(1)\n        self.head = nn.Linear(256, num_classes)\n        self.apply(self._init)\n\n    @staticmethod\n    def _init(m):\n        if isinstance(m, nn.Conv2d):\n            nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n        elif isinstance(m, nn.BatchNorm2d):\n            nn.init.ones_(m.weight); nn.init.zeros_(m.bias)\n        elif isinstance(m, nn.Linear):\n            nn.init.normal_(m.weight, 0, 0.01); nn.init.zeros_(m.bias)\n\n    def forward(self, x):\n        x = self.stem(x)\n        x = self.s1(x); x = self.s2(x); x = self.s3(x)\n        x = self.pool(x).flatten(1)\n        return self.head(x)\n\nmodel = SmallResNetCIFAR(num_classes=10).to(DEVICE)\nsum(p.numel() for p in model.parameters()), model.__class__.__name__","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T14:39:50.937635Z","iopub.execute_input":"2025-10-15T14:39:50.938046Z","iopub.status.idle":"2025-10-15T14:39:51.167847Z","shell.execute_reply.started":"2025-10-15T14:39:50.938021Z","shell.execute_reply":"2025-10-15T14:39:51.167113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# D) AdamW + warmup→cosine (Challenge requirement)\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import LinearLR, CosineAnnealingLR, SequentialLR\n\nEPOCHS        = 40\nBASE_LR       = 3e-3\nMIN_LR_FACTOR = 0.01\nWARMUP_EPOCHS = 5\nWEIGHT_DECAY  = 5e-4\n\noptimizer = optim.AdamW(model.parameters(), lr=BASE_LR, betas=(0.9,0.999), weight_decay=WEIGHT_DECAY)\n\nwarmup = LinearLR(optimizer, start_factor=0.01, end_factor=1.0, total_iters=WARMUP_EPOCHS)\ncosine = CosineAnnealingLR(optimizer, T_max=max(1, EPOCHS-WARMUP_EPOCHS), eta_min=BASE_LR*MIN_LR_FACTOR)\nscheduler = SequentialLR(optimizer, schedulers=[warmup, cosine], milestones=[WARMUP_EPOCHS])\n\nprint(\"Using AdamW + Linear warmup → CosineAnnealingLR\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T14:39:51.169422Z","iopub.execute_input":"2025-10-15T14:39:51.169667Z","iopub.status.idle":"2025-10-15T14:39:51.177090Z","shell.execute_reply.started":"2025-10-15T14:39:51.169647Z","shell.execute_reply":"2025-10-15T14:39:51.176339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# E) Train with CutMix + AMP (torch.amp) and validate — CORRECTED\n\nimport math, random\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.amp import GradScaler, autocast  # NEW API\n\ncriterion = nn.CrossEntropyLoss()\n\n# ----- AMP setup (works on GPU/CPU) -----\nAMP_ENABLED = (DEVICE.type == \"cuda\")\nscaler = GradScaler(\"cuda\" if AMP_ENABLED else \"cpu\")\n\n# ----- CutMix helpers -----\nUSE_CUTMIX   = True\nCUTMIX_ALPHA = 1.0\nCUTMIX_P     = 0.5\n\ndef rand_bbox(W, H, lam):\n    cut_w = int(W * math.sqrt(1 - lam))\n    cut_h = int(H * math.sqrt(1 - lam))\n    cx = random.randint(0, W)\n    cy = random.randint(0, H)\n    x1 = max(cx - cut_w // 2, 0); y1 = max(cy - cut_h // 2, 0)\n    x2 = min(cx + cut_w // 2, W); y2 = min(cy + cut_h // 2, H)\n    return x1, y1, x2, y2\n\ndef apply_cutmix(x, y, alpha=1.0):\n    if alpha <= 0:\n        return x, y, y, 1.0\n    lam = np.random.beta(alpha, alpha)\n    B, _, H, W = x.size()\n    index = torch.randperm(B, device=x.device)\n    x_shuf, y_shuf = x[index], y[index]\n    x1, y1, x2, y2 = rand_bbox(W, H, lam)\n    x[:, :, y1:y2, x1:x2] = x_shuf[:, :, y1:y2, x1:x2]\n    lam = 1.0 - ((x2 - x1) * (y2 - y1) / (W * H))\n    return x, y, y_shuf, lam\n\n@torch.no_grad()\ndef evaluate(loader, use_model=None):\n    m = model if use_model is None else use_model\n    m.eval()\n    correct, total, loss_sum = 0, 0, 0.0\n    with autocast(\"cuda\", dtype=torch.float16, enabled=AMP_ENABLED):  # NEW\n        for x, y in loader:\n            x = x.to(DEVICE, non_blocking=True)\n            y = y.to(DEVICE, non_blocking=True)\n            logits = m(x)\n            loss = criterion(logits, y)\n            loss_sum += loss.item() * x.size(0)\n            pred = logits.argmax(1)\n            correct += (pred == y).sum().item()\n            total += x.size(0)\n    return loss_sum / total, correct / total\n\nbest_val = 0.0\n\nfor epoch in range(1, EPOCHS + 1):\n    model.train()\n    for x, y in train_loader:\n        x = x.to(DEVICE, non_blocking=True)\n        y = y.to(DEVICE, non_blocking=True)\n\n        use_cm = USE_CUTMIX and (random.random() < CUTMIX_P)\n        if use_cm:\n            x, y_a, y_b, lam = apply_cutmix(x, y, alpha=CUTMIX_ALPHA)\n\n        optimizer.zero_grad(set_to_none=True)\n        with autocast(\"cuda\", dtype=torch.float16, enabled=AMP_ENABLED):  # NEW\n            logits = model(x)\n            if use_cm:\n                loss = lam * criterion(logits, y_a) + (1.0 - lam) * criterion(logits, y_b)\n            else:\n                loss = criterion(logits, y)\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n    scheduler.step()\n\n    vloss, vacc = evaluate(val_loader)\n    print(f\"Epoch {epoch:03d} | val_acc={vacc:.4f} | val_loss={vloss:.4f} | lr={scheduler.get_last_lr()[0]:.5f}\")\n\n    if vacc > best_val:\n        best_val = vacc\n        torch.save(model.state_dict(), \"best_model.pth\")\n\nprint(f\"Best val acc: {best_val:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T14:39:51.178031Z","iopub.execute_input":"2025-10-15T14:39:51.178383Z","execution_failed":"2025-10-15T14:49:15.032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# F) Offline evaluation on the official CIFAR-10 test set (requires Internet=ON)\nfrom torchvision.datasets import CIFAR10\nfrom torch.utils.data import DataLoader\nfrom torch.amp import autocast\nimport torch\nimport torch.nn as nn\n\n# in case it's not defined above:\nAMP_ENABLED = (DEVICE.type == \"cuda\")\n\n# dataset + loader\ntest_official = CIFAR10(\n    root=\"/kaggle/working/data\",\n    train=False,\n    download=True,\n    transform=tfm_test,      # uses the same normalization as train/val\n)\ntest_loader_official = DataLoader(\n    test_official, batch_size=BS, shuffle=False, num_workers=2, pin_memory=True\n)\n\n# load best checkpoint and eval\nmodel.load_state_dict(torch.load(\"best_model.pth\", map_location=DEVICE))\nmodel.eval()\n\ncriterion = nn.CrossEntropyLoss()\nloss_sum, correct, total = 0.0, 0, 0\n\nwith torch.no_grad():\n    for x, y in test_loader_official:\n        x = x.to(DEVICE, non_blocking=True)\n        y = y.to(DEVICE, non_blocking=True)\n\n        # NEW AMP API\n        with autocast(\"cuda\", dtype=torch.float16, enabled=AMP_ENABLED):\n            logits = model(x)\n            loss = criterion(logits, y)\n\n        loss_sum += loss.item() * x.size(0)\n        pred = logits.argmax(1)\n        correct += (pred == y).sum().item()\n        total += x.size(0)\n\nta = correct / total\ntl = loss_sum / total\nprint(f\"Official CIFAR-10 test accuracy (≈ Kaggle score): {ta:.4f}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-15T14:49:15.035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# G) Predict Kaggle test set and create submission.csv\nfrom torch.utils.data import DataLoader\nfrom torch.amp import autocast\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport glob\nimport torch\n\n# in case it's not defined above:\nAMP_ENABLED = (DEVICE.type == \"cuda\")\n\n# gather test ids in numeric order\ntest_paths = sorted(glob.glob(test_out + \"/*.png\"), key=lambda p: int(Path(p).stem))\ntest_ids = pd.DataFrame({\"id\": [int(Path(p).stem) for p in test_paths]})\n\n# dataset/loader (uses the same CifarFiles and tfm_test defined earlier)\ntest_ds = CifarFiles(test_ids, test_out, tfm_test)\ntest_loader = DataLoader(test_ds, batch_size=BS, shuffle=False, num_workers=2, pin_memory=True)\n\n# load best model and eval\nmodel.load_state_dict(torch.load(\"best_model.pth\", map_location=DEVICE))\nmodel.eval()\n\npreds = []\nwith torch.no_grad():\n    for x, idx in test_loader:\n        x = x.to(DEVICE, non_blocking=True)\n        # NEW AMP API\n        with autocast(\"cuda\", dtype=torch.float16, enabled=AMP_ENABLED):\n            logits = model(x)\n        preds.append(logits.argmax(1).cpu().numpy())\n\npreds = np.concatenate(preds)\n\n# map indices -> class names and save\nsubmission = pd.DataFrame({\n    \"id\": test_ids[\"id\"].values,\n    \"label\": [classes[i] for i in preds]\n})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(submission.head(), \"\\nSaved: submission.csv\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-15T14:49:15.036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = []  # put this before the training loop\n\n# inside your epoch loop, after you compute vacc/vloss:\nhistory.append((epoch, vacc, vloss, scheduler.get_last_lr()[0]))\n\n# after training:\nprint(\"epoch | val_acc | val_loss | lr\")\nfor e, a, l, lr in history:\n    print(f\"{e:03d}   | {a:.4f}  | {l:.4f}  | {lr:.5f}\")\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-15T14:49:15.036Z"}},"outputs":[],"execution_count":null}]}