{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================\n# CIFAR-10: Single-Cell Train → Infer → Submit\n# - Custom ResNet-like + SE (no off-the-shelf)\n# - AdamW + OneCycleLR (non-basic)\n# - RandAugment + MixUp/CutMix + RandomErasing (non-basic)\n# - AMP + channels_last\n# - Extract 300k test.7z → ./test, infer, write submission.csv\n# ============================================\nimport os, sys, time, math, random, warnings, glob, shutil, subprocess, tempfile\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport torch, torch.nn as nn, torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Subset\nfrom torchvision import datasets, transforms\nfrom torch.cuda.amp import autocast, GradScaler\n\n# -----------------------\n# Repro + device\n# -----------------------\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntorch.backends.cudnn.benchmark = True\nprint(\"Device:\", device)\n\n# -----------------------\n# Data & augmentation\n# -----------------------\nCIFAR_MEAN=(0.4914,0.4822,0.4465)\nCIFAR_STD =(0.2470,0.2435,0.2616)\n\n# Try RandAugment; fall back to AutoAugment(CIFAR10)\ntry:\n    from torchvision.transforms import RandAugment\n    extra_aug = RandAugment(num_ops=2, magnitude=9)\nexcept Exception:\n    from torchvision.transforms import AutoAugment, AutoAugmentPolicy\n    extra_aug = AutoAugment(AutoAugmentPolicy.CIFAR10)\n\ntrain_tfms = transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),\n    extra_aug,\n    transforms.ToTensor(),\n    transforms.Normalize(CIFAR_MEAN, CIFAR_STD),\n    transforms.RandomErasing(p=0.25, scale=(0.02,0.2), ratio=(0.3,3.3))\n])\ntest_tfms  = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\nDATA_ROOT=\"/kaggle/working\"\n_base = datasets.CIFAR10(root=DATA_ROOT, train=True, download=True, transform=None)\nidx = np.arange(len(_base)); np.random.default_rng(SEED).shuffle(idx)\nVAL_SZ=5000\ntrain_idx, val_idx = idx[VAL_SZ:], idx[:VAL_SZ]\n\ntrain_set = Subset(datasets.CIFAR10(root=DATA_ROOT, train=True, download=False, transform=train_tfms), train_idx.tolist())\nval_set   = Subset(datasets.CIFAR10(root=DATA_ROOT, train=True, download=False, transform=test_tfms),  val_idx.tolist())\n\nBATCH_TRAIN, BATCH_VAL = 256, 512\ntrain_loader = DataLoader(train_set, batch_size=BATCH_TRAIN, shuffle=True,  num_workers=2, pin_memory=True, drop_last=True)\nval_loader   = DataLoader(val_set,   batch_size=BATCH_VAL,   shuffle=False, num_workers=2, pin_memory=True)\n\nprint(f\"Train rows: {len(train_set)} | Val rows: {len(val_set)}\")\n\n# -----------------------\n# Custom model (ResBlock + SE)\n# -----------------------\nclass SE(nn.Module):\n    def __init__(self, c, r=16):\n        super().__init__()\n        self.fc1 = nn.Conv2d(c, c//r, 1)\n        self.fc2 = nn.Conv2d(c//r, c, 1)\n    def forward(self, x):\n        s = x.mean(dim=(2,3), keepdim=True)\n        s = F.relu(self.fc1(s), inplace=True)\n        s = torch.sigmoid(self.fc2(s))\n        return x * s\n\nclass ResBlock(nn.Module):\n    def __init__(self, in_c, out_c, stride=1):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_c, out_c, 3, stride=stride, padding=1, bias=False)\n        self.bn1   = nn.BatchNorm2d(out_c)\n        self.conv2 = nn.Conv2d(out_c, out_c, 3, padding=1, bias=False)\n        self.bn2   = nn.BatchNorm2d(out_c)\n        self.se    = SE(out_c, r=16)\n        self.skip  = (nn.Identity() if (in_c==out_c and stride==1)\n                      else nn.Sequential(nn.Conv2d(in_c, out_c, 1, stride=stride, bias=False),\n                                         nn.BatchNorm2d(out_c)))\n    def forward(self, x):\n        s = self.skip(x)\n        x = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        x = self.bn2(self.conv2(x))\n        x = self.se(x)\n        x = F.relu(x + s, inplace=True)\n        return x\n\nclass CifarSEnet(nn.Module):\n    def __init__(self, num_classes=10, width=64):\n        super().__init__()\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, width, 3, padding=1, bias=False),\n            nn.BatchNorm2d(width),\n            nn.ReLU(inplace=True)\n        )\n        self.layer1 = nn.Sequential(ResBlock(width, width, 1), ResBlock(width, width, 1))\n        self.layer2 = nn.Sequential(ResBlock(width, width*2, 2), ResBlock(width*2, width*2, 1))\n        self.layer3 = nn.Sequential(ResBlock(width*2, width*4, 2), ResBlock(width*4, width*4, 1))\n        self.head   = nn.Sequential(nn.AdaptiveAvgPool2d(1), nn.Flatten(), nn.Linear(width*4, num_classes))\n        self.apply(self._init)\n    @staticmethod\n    def _init(m):\n        if isinstance(m, nn.Conv2d):\n            nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n        elif isinstance(m, nn.BatchNorm2d):\n            nn.init.ones_(m.weight); nn.init.zeros_(m.bias)\n        elif isinstance(m, nn.Linear):\n            nn.init.normal_(m.weight, 0, 0.01); nn.init.zeros_(m.bias)\n    def forward(self, x):\n        x = self.stem(x)\n        x = self.layer1(x); x = self.layer2(x); x = self.layer3(x)\n        x = self.head(x)\n        return x\n\nmodel = CifarSEnet(num_classes=10, width=64).to(device).to(memory_format=torch.channels_last)\n\n# -----------------------\n# MixUp / CutMix utilities\n# -----------------------\ndef rand_bbox(W, H, lam):\n    cut_rat = math.sqrt(1. - lam)\n    cut_w, cut_h = int(W * cut_rat), int(H * cut_rat)\n    cx, cy = np.random.randint(W), np.random.randint(H)\n    x1 = np.clip(cx - cut_w // 2, 0, W)\n    y1 = np.clip(cy - cut_h // 2, 0, H)\n    x2 = np.clip(cx + cut_w // 2, 0, W)\n    y2 = np.clip(cy + cut_h // 2, 0, H)\n    return x1, y1, x2, y2\n\ndef apply_mixup_cutmix(x, y, p_mixup=0.5, p_cutmix=0.5, alpha_mix=0.4, alpha_cut=1.0):\n    do_mixup  = np.random.rand() < p_mixup\n    do_cutmix = (not do_mixup) and (np.random.rand() < p_cutmix)\n    if do_mixup:\n        lam = np.random.beta(alpha_mix, alpha_mix)\n        idx = torch.randperm(x.size(0), device=x.device)\n        x = lam * x + (1 - lam) * x[idx]\n        y_a, y_b = y, y[idx]\n        return x, y_a, y_b, lam, \"mixup\"\n    elif do_cutmix:\n        lam = np.random.beta(alpha_cut, alpha_cut)\n        idx = torch.randperm(x.size(0), device=x.device)\n        x1,y1,x2,y2 = rand_bbox(x.size(3), x.size(2), lam)\n        x[:, :, y1:y2, x1:x2] = x[idx, :, y1:y2, x1:x2]\n        lam = 1 - ((x2-x1)*(y2-y1) / (x.size(-1)*x.size(-2)))\n        y_a, y_b = y, y[idx]\n        return x, y_a, y_b, lam, \"cutmix\"\n    else:\n        return x, y, y, 1.0, \"none\"\n\n# -----------------------\n# Optimizer / LR / EMA / Loss / AMP\n# -----------------------\nEPOCHS = 60\nBASE_LR = 5e-3\nWEIGHT_DECAY = 5e-4\n\nopt = torch.optim.AdamW(model.parameters(), lr=BASE_LR, weight_decay=WEIGHT_DECAY)\nsteps_per_epoch = len(train_loader)\nsched = torch.optim.lr_scheduler.OneCycleLR(\n    opt, max_lr=BASE_LR, epochs=EPOCHS, steps_per_epoch=steps_per_epoch,\n    pct_start=0.2, div_factor=10.0, final_div_factor=1e3, anneal_strategy='cos'\n)\ncrit = nn.CrossEntropyLoss(label_smoothing=0.05)\nscaler = GradScaler()\n\n# Simple EMA of weights (optional but helpful)\nclass EMA:\n    def __init__(self, model, decay=0.999):\n        self.decay = decay\n        self.shadow = {k: v.detach().clone() for k,v in model.state_dict().items() if v.dtype.is_floating_point}\n    @torch.no_grad()\n    def update(self, model):\n        for k, v in model.state_dict().items():\n            if k in self.shadow and v.dtype.is_floating_point:\n                self.shadow[k].mul_(self.decay).add_(v.detach(), alpha=1-self.decay)\n    def load_shadow(self, model):\n        state = model.state_dict()\n        for k, v in self.shadow.items():\n            state[k].copy_(v)\n\nema = EMA(model, decay=0.999)\n\n# -----------------------\n# Train / Eval loops\n# -----------------------\ndef run_epoch(dl, train=True):\n    model.train(train)\n    total = correct = loss_sum = 0.0\n    for (xb, yb) in dl:\n        xb = xb.to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n        yb = yb.to(device, non_blocking=True)\n\n        if train:\n            xb_aug, ya, yb2, lam, mode = apply_mixup_cutmix(xb, yb)  # mix/cut/none\n            opt.zero_grad(set_to_none=True)\n            with autocast():\n                logits = model(xb_aug)\n                if mode == \"none\":\n                    loss = crit(logits, ya)\n                    pred  = logits.argmax(1)\n                    correct += (pred == ya).sum().item()\n                else:\n                    loss = lam*crit(logits, ya) + (1-lam)*crit(logits, yb2)\n                    # for accuracy, use primary labels as proxy\n                    pred = logits.argmax(1)\n                    correct += (pred == ya).sum().item()*lam + (pred == yb2).sum().item()*(1-lam)\n            scaler.scale(loss).backward()\n            scaler.step(opt)\n            scaler.update()\n            sched.step()\n            ema.update(model)\n            loss_sum += loss.item() * xb.size(0)\n            total    += xb.size(0)\n        else:\n            with torch.no_grad(), autocast():\n                logits = model(xb)\n                loss = crit(logits, yb)\n                pred = logits.argmax(1)\n            loss_sum += loss.item() * xb.size(0)\n            correct  += (pred == yb).sum().item()\n            total    += xb.size(0)\n\n    return loss_sum/total, correct/total\n\nbest_acc = 0.0\nbest_state = None\n\nfor e in range(1, EPOCHS+1):\n    tr_loss, tr_acc = run_epoch(train_loader, True)\n    # evaluate with EMA weights for stability\n    saved = {k:v.detach().clone() for k,v in model.state_dict().items()}\n    ema.load_shadow(model)\n    va_loss, va_acc = run_epoch(val_loader, False)\n    model.load_state_dict(saved)  # restore train weights\n\n    if va_acc > best_acc:\n        best_acc = va_acc\n        best_state = {k:v.detach().cpu() for k,v in model.state_dict().items()}\n    print(f\"Ep{e:02d}/{EPOCHS}  lr={sched.get_last_lr()[0]:.5f}  train_acc={tr_acc:.3f}  val_acc(EMA)={va_acc:.3f}\")\n\ntorch.save(best_state, \"best_custom.pt\")\nprint(f\"✅ Saved best_custom.pt (best val_acc={best_acc:.4f})\")\n\n# -----------------------\n# Extract Kaggle test set (300k) to ./test once\n# -----------------------\nCOMP_DIR = \"/kaggle/input/cifar-10\"\nTEST_7Z  = os.path.join(COMP_DIR, \"test.7z\")\nTEST_DIR = \"./test\"\nTOTAL    = 300_000\nassert os.path.exists(TEST_7Z), \"Missing /kaggle/input/cifar-10/test.7z\"\n\ndef ensure_7z():\n    if shutil.which(\"7z\"): return \"7z\"\n    subprocess.run([\"apt-get\",\"update\",\"-y\"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n    subprocess.run([\"apt-get\",\"install\",\"-y\",\"p7zip-full\"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n    assert shutil.which(\"7z\"), \"7z not found after install\"\n    return \"7z\"\n\nos.makedirs(TEST_DIR, exist_ok=True)\nhave = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\nif have >= TOTAL:\n    print(f\"✅ Already extracted: {have}/{TOTAL}\")\nelse:\n    sevenz = ensure_7z()\n    print(f\"Extracting test set to {os.path.abspath(TEST_DIR)} ...\")\n    proc = subprocess.Popen([sevenz, \"x\", TEST_7Z, \"-y\", \"-o.\"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n    last = -1\n    while proc.poll() is None:\n        done = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\n        if done != last:\n            pct = 100.0 * min(done, TOTAL) / TOTAL\n            print(f\"\\rprogress: {done}/{TOTAL} ({pct:5.1f}%)\", end=\"\")\n            last = done\n        time.sleep(1)\n    done = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\n    print(f\"\\rprogress: {done}/{TOTAL} (100.0%)\")\n    print(f\"✅ Extracted: {done}/{TOTAL}\")\n\n# -----------------------\n# Inference on test → submission.csv\n# -----------------------\nimport pandas as pd\nfrom PIL import Image\n\nCLASS_NAMES = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\nSAMPLE_SUB  = os.path.join(COMP_DIR, \"sampleSubmission.csv\")\nassert os.path.exists(SAMPLE_SUB), \"Missing sampleSubmission.csv\"\nsample = pd.read_csv(SAMPLE_SUB)\nids = sample[\"id\"].astype(int).tolist()\nN = len(ids)\nprint(\"IDs to predict:\", N)\n\n# Rebuild model and load best weights (EMA-evaluated checkpoint)\nmodel_inf = CifarSEnet(num_classes=10, width=64).to(device).to(memory_format=torch.channels_last)\nstate = torch.load(\"best_custom.pt\", map_location=\"cpu\")\nmodel_inf.load_state_dict(state)\nmodel_inf.eval()\n\neval_tfm = transforms.Compose([transforms.ToTensor(), transforms.Normalize(CIFAR_MEAN, CIFAR_STD)])\n\nBATCH = 1024\npred_idx = np.empty(N, dtype=np.int64)\n\ndef batched(iterable, n):\n    for i in range(0, len(iterable), n):\n        yield i, iterable[i:i+n]\n\nwith torch.no_grad():\n    done=0\n    for start, id_batch in batched(ids, BATCH):\n        imgs=[]\n        for i in id_batch:\n            p = os.path.join(TEST_DIR, f\"{i}.png\")\n            img = Image.open(p).convert(\"RGB\")\n            imgs.append(eval_tfm(img))\n        xb = torch.stack(imgs).to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n        with autocast():\n            logits = model_inf(xb)\n        preds = logits.argmax(1).cpu().numpy()\n        pred_idx[start:start+len(id_batch)] = preds\n        done += len(id_batch)\n        if done % 10000 == 0:\n            print(f\"progress: {done}/{N}\")\n\n# Build submission\nlabels = [CLASS_NAMES[i] for i in pred_idx.tolist()]\nsub = pd.DataFrame({\"id\": np.array(ids, dtype=np.int32), \"label\": labels})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"✅ Wrote submission.csv\")\nprint(sub.head())\nprint(\"shape:\", sub.shape, \"| ids unique:\", sub['id'].is_unique)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Extra Cell: Export standalone submission(s)\n# - Ensures a clean submission.csv\n# - Also writes a timestamped copy for easy download\n# ================================\nimport os, time, numpy as np, pandas as pd\n\nCLASS_NAMES = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\n\ndef build_from_npy(ids_path=\"ids.npy\", pred_path=\"pred_idx.npy\"):\n    ids = np.load(ids_path)\n    pred_idx = np.load(pred_path)\n    labels = [CLASS_NAMES[i] for i in pred_idx.tolist()]\n    return pd.DataFrame({\"id\": ids.astype(int), \"label\": labels})\n\n# Prefer rebuilding from npy artifacts if they exist; else reuse existing CSV\nif os.path.exists(\"ids.npy\") and os.path.exists(\"pred_idx.npy\"):\n    sub = build_from_npy()\nelse:\n    sub = pd.read_csv(\"submission.csv\")  # fallback if already created\n\n# Write main file\nsub.to_csv(\"submission.csv\", index=False)\n\n# Also write a timestamped standalone copy (shows up separately in Kaggle Output)\nstamp = time.strftime(\"%Y%m%d-%H%M%S\")\nstandalone_name = f\"submission_standalone_{stamp}.csv\"\nsub.to_csv(standalone_name, index=False)\n\nprint(\"✅ Wrote:\", \"submission.csv\", \"and\", standalone_name)\nprint(\"Tip: Both files appear in the Kaggle 'Output' tab; you can download either one directly.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}