{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":3649}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q \"py7zr<1.0\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-23T05:54:17.585975Z","iopub.execute_input":"2026-09-23T05:54:17.586805Z","iopub.status.idle":"2026-09-23T05:54:21.145588Z","shell.execute_reply.started":"2026-09-23T05:54:17.586763Z","shell.execute_reply":"2026-09-23T05:54:21.144681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# CIFAR-10 Kaggle competition — ResNet-9 trained from scratch\n#   * Uses ONLY the attached competition files (train.7z, trainLabels.csv, test.7z)\n#     -> no internet download needed\n#   * Phase 1: train on 45,000 / validate on 5,000  (estimates the Kaggle score)\n#   * Phase 2: retrain from a fresh random init on all 50,000, predict 300,000\n#     Kaggle test images, write submission.csv\n#   * Weights are randomly initialized (no pre-trained weights)\n#   * Archives are read straight into memory with py7zr (no 300k files on disk)\n#\n# Run this first, in its own cell (needs Internet on once):\n#   !pip install -q \"py7zr<1.0\"\n# ============================================================\nimport numpy as np\nimport pandas as pd\nimport os, io, glob, time\nfrom multiprocessing import Pool\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# ---------------- Config ----------------\nSEED = 42\nEPOCHS = 40\nBATCH_SIZE = 512\nMAX_LR = 0.2\nWEIGHT_DECAY = 5e-4\nLABEL_SMOOTHING = 0.1\nCUTOUT = 8\nRUN_VALIDATION = True      # set False to skip phase 1 and only train the final model\nVAL_SIZE = 5000            # 500 images per class\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nN_GPUS = torch.cuda.device_count()\nuse_amp = DEVICE == \"cuda\"\nWORK = \"/kaggle/working\" if os.path.isdir(\"/kaggle/working\") else \".\"\nCLASSES = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\",\n           \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\n\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.backends.cudnn.benchmark = True\nprint(f\"Device: {DEVICE} | GPUs: {N_GPUS}\")\n\n\n# ---------------- File helpers ----------------\ndef find_input(name):\n    hits = glob.glob(f\"/kaggle/input/**/{name}\", recursive=True)\n    if not hits:\n        raise FileNotFoundError(f\"{name} not found in /kaggle/input — attach the CIFAR-10 competition data\")\n    return hits[0]\n\n\ndef find_cache(name):\n    \"\"\"Cached .npz from a previous run (attached as input) or in the working dir.\"\"\"\n    hits = glob.glob(f\"/kaggle/input/**/{name}\", recursive=True)\n    local = os.path.join(WORK, name)\n    if os.path.exists(local):\n        hits.append(local)\n    return hits[0] if hits else None\n\n\ndef _decode_png(b):\n    return np.asarray(Image.open(io.BytesIO(b)).convert(\"RGB\"), dtype=np.uint8).transpose(2, 0, 1)\n\n\ndef load_7z_pngs(archive):\n    \"\"\"Decompress a .7z of PNGs in memory and decode them — nothing is written to disk.\"\"\"\n    import py7zr\n    t = time.time()\n    print(f\"Reading {archive} into memory ...\")\n    with py7zr.SevenZipFile(archive, \"r\") as z:\n        entries = z.readall()                     # {name: BytesIO}\n    names = [n for n in entries if n.endswith(\".png\")]\n    names.sort(key=lambda n: int(os.path.basename(n)[:-4]))\n    ids = np.array([int(os.path.basename(n)[:-4]) for n in names])\n    blobs = [entries[n].getvalue() for n in names]\n    del entries\n    print(f\"  read {len(names)} files in {time.time()-t:.0f}s, decoding ...\")\n    with Pool(os.cpu_count()) as pool:\n        imgs = pool.map(_decode_png, blobs, chunksize=2000)\n    print(f\"  done in {time.time()-t:.0f}s\")\n    return np.stack(imgs), ids\n\n\ndef load_train():\n    cache = find_cache(\"cifar10_train.npz\")\n    if cache:\n        print(\"Using cached training data:\", cache)\n        d = np.load(cache)\n        return d[\"X\"], d[\"y\"]\n    X, ids = load_7z_pngs(find_input(\"train.7z\"))\n    labels = pd.read_csv(find_input(\"trainLabels.csv\")).set_index(\"id\").loc[ids, \"label\"]\n    y = labels.map({c: i for i, c in enumerate(CLASSES)}).to_numpy().astype(np.int64)\n    np.savez(os.path.join(WORK, \"cifar10_train.npz\"), X=X, y=y)\n    return X, y\n\n\ndef load_kaggle_test():\n    cache = find_cache(\"kaggle_test.npz\")\n    if cache:\n        print(\"Using cached Kaggle test data:\", cache)\n        d = np.load(cache)\n        return d[\"X\"], d[\"ids\"]\n    X, ids = load_7z_pngs(find_input(\"test.7z\"))\n    np.savez(os.path.join(WORK, \"kaggle_test.npz\"), X=X, ids=ids)\n    return X, ids\n\n\n# ---------------- Load data ----------------\nt0 = time.time()\nX_all, y_all = load_train()\nX_test, test_ids = load_kaggle_test()\nprint(f\"Train: {X_all.shape} | Kaggle test: {X_test.shape} | {time.time()-t0:.0f}s\")\nassert len(X_all) == 50000 and len(X_test) == 300000\n\n# Normalization constants computed from the training images (not weights)\nmean = torch.tensor(X_all.mean(axis=(0, 2, 3), dtype=np.float64) / 255,\n                    device=DEVICE, dtype=torch.float32).view(1, 3, 1, 1)\nstd = torch.tensor(X_all.std(axis=(0, 2, 3), dtype=np.float64) / 255,\n                   device=DEVICE, dtype=torch.float32).view(1, 3, 1, 1)\nprint(\"mean\", mean.flatten().tolist(), \"std\", std.flatten().tolist())\n\n\ndef to_gpu(x_uint8):\n    x = torch.from_numpy(np.ascontiguousarray(x_uint8)).to(DEVICE).float().div_(255)\n    return (x - mean) / std\n\n\n# ---------------- GPU augmentation ----------------\ndef augment(x):\n    n = x.size(0)\n    xp = F.pad(x, (4, 4, 4, 4), mode=\"reflect\")            # random crop\n    dy = torch.randint(0, 9, (n,), device=x.device)\n    dx = torch.randint(0, 9, (n,), device=x.device)\n    base = torch.arange(32, device=x.device)\n    rows = (dy[:, None] + base)[:, :, None]\n    cols = (dx[:, None] + base)[:, None, :]\n    idx = torch.arange(n, device=x.device)[:, None, None]\n    x = xp.permute(0, 2, 3, 1)[idx, rows, cols].permute(0, 3, 1, 2)\n    flip = torch.rand(n, device=x.device) < 0.5            # horizontal flip\n    x = torch.where(flip[:, None, None, None], x.flip(3), x)\n    if CUTOUT > 0:                                         # cutout\n        cy = torch.randint(0, 32, (n, 1, 1), device=x.device)\n        cx = torch.randint(0, 32, (n, 1, 1), device=x.device)\n        mask = ((base[None, :, None] - cy).abs() < CUTOUT // 2) & \\\n               ((base[None, None, :] - cx).abs() < CUTOUT // 2)\n        x = x.masked_fill(mask[:, None], 0.0)\n    return x.contiguous(memory_format=torch.channels_last)\n\n\n# ---------------- Model: ResNet-9 (random init) ----------------\ndef conv_bn(cin, cout, pool=False):\n    layers = [nn.Conv2d(cin, cout, 3, padding=1, bias=False),\n              nn.BatchNorm2d(cout), nn.ReLU(inplace=True)]\n    if pool:\n        layers.append(nn.MaxPool2d(2))\n    return nn.Sequential(*layers)\n\n\nclass Residual(nn.Module):\n    def __init__(self, c):\n        super().__init__()\n        self.body = nn.Sequential(conv_bn(c, c), conv_bn(c, c))\n    def forward(self, x):\n        return x + self.body(x)\n\n\nclass ResNet9(nn.Module):\n    def __init__(self, num_classes=10):\n        super().__init__()\n        self.net = nn.Sequential(\n            conv_bn(3, 64),\n            conv_bn(64, 128, pool=True), Residual(128),\n            conv_bn(128, 256, pool=True),\n            conv_bn(256, 512, pool=True), Residual(512),\n            nn.AdaptiveMaxPool2d(1), nn.Flatten(),\n            nn.Linear(512, num_classes, bias=False),\n        )\n    def forward(self, x):\n        return self.net(x) * 0.125\n\n\ndef init_weights(m):\n    if isinstance(m, (nn.Conv2d, nn.Linear)):\n        nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n    elif isinstance(m, nn.BatchNorm2d):\n        nn.init.ones_(m.weight); nn.init.zeros_(m.bias)\n\n\ndef make_model():\n    m = ResNet9()\n    m.apply(init_weights)                                  # explicit random init\n    m = m.to(DEVICE).to(memory_format=torch.channels_last)\n    if N_GPUS > 1:\n        m = nn.DataParallel(m)                             # use both T4s\n    return m\n\n\ndef unwrap(m):\n    return m.module if isinstance(m, nn.DataParallel) else m\n\n\n# ---------------- Predict (with optional flip TTA) ----------------\n@torch.no_grad()\ndef predict(model, X_uint8, tta=True, bs=2000):\n    model.eval()\n    out = []\n    for i in range(0, len(X_uint8), bs):\n        x = to_gpu(X_uint8[i:i + bs]).contiguous(memory_format=torch.channels_last)\n        with torch.autocast(\"cuda\", dtype=torch.float16, enabled=use_amp):\n            logits = model(x)\n            if tta:\n                logits = logits + model(x.flip(3))\n        out.append(logits.argmax(1).cpu())\n    return torch.cat(out).numpy()\n\n\n# ---------------- Training ----------------\ndef train(X_tr, y_tr, X_val=None, y_val=None, tag=\"\"):\n    torch.manual_seed(SEED)\n    Xg = to_gpu(X_tr)\n    Yg = torch.from_numpy(y_tr).long().to(DEVICE)\n    model = make_model()\n\n    decay, no_decay = [], []\n    for _, p in model.named_parameters():\n        (no_decay if p.ndim <= 1 else decay).append(p)\n    optimizer = torch.optim.SGD(\n        [{\"params\": decay, \"weight_decay\": WEIGHT_DECAY},\n         {\"params\": no_decay, \"weight_decay\": 0.0}],\n        lr=MAX_LR, momentum=0.9, nesterov=True)\n    steps = (len(Xg) + BATCH_SIZE - 1) // BATCH_SIZE\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n        optimizer, max_lr=MAX_LR, epochs=EPOCHS, steps_per_epoch=steps,\n        pct_start=0.25, div_factor=20, final_div_factor=200)\n    criterion = nn.CrossEntropyLoss(label_smoothing=LABEL_SMOOTHING)\n    scaler = torch.amp.GradScaler(\"cuda\", enabled=use_amp)\n\n    print(f\"\\n===== Training [{tag}] on {len(Xg)} images =====\")\n    for epoch in range(1, EPOCHS + 1):\n        model.train()\n        t0 = time.time()\n        perm = torch.randperm(len(Xg), device=DEVICE)\n        tot_loss, tot_correct = 0.0, 0\n        for i in range(0, len(Xg), BATCH_SIZE):\n            idx = perm[i:i + BATCH_SIZE]\n            x, y = augment(Xg[idx]), Yg[idx]\n            with torch.autocast(\"cuda\", dtype=torch.float16, enabled=use_amp):\n                out = model(x)\n                loss = criterion(out, y)\n            optimizer.zero_grad(set_to_none=True)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            scheduler.step()\n            tot_loss += loss.item() * len(idx)\n            tot_correct += (out.argmax(1) == y).sum().item()\n        msg = (f\"Epoch {epoch:2d}/{EPOCHS} | loss {tot_loss/len(Xg):.4f} | \"\n               f\"train acc {tot_correct/len(Xg):.4f}\")\n        if X_val is not None:\n            msg += f\" | val acc {(predict(model, X_val, tta=False) == y_val).mean():.4f}\"\n        print(msg + f\" | {time.time()-t0:.1f}s\")\n    del Xg, Yg\n    return model\n\n\n# ---------------- Phase 1: 45k train / 5k validation ----------------\nif RUN_VALIDATION:\n    rng = np.random.default_rng(SEED)\n    val_idx = np.concatenate([\n        rng.choice(np.where(y_all == c)[0], VAL_SIZE // 10, replace=False)\n        for c in range(10)])                                  # stratified: 500 per class\n    tr_mask = np.ones(len(y_all), dtype=bool)\n    tr_mask[val_idx] = False\n\n    val_model = train(X_all[tr_mask], y_all[tr_mask],\n                      X_all[val_idx], y_all[val_idx], tag=\"validation 45k/5k\")\n    acc_single = (predict(val_model, X_all[val_idx], tta=False) == y_all[val_idx]).mean()\n    acc_tta = (predict(val_model, X_all[val_idx], tta=True) == y_all[val_idx]).mean()\n    print(f\"\\nValidation accuracy: {acc_single*100:.2f}% (single view) | \"\n          f\"{acc_tta*100:.2f}% (flip TTA)\")\n    print(\"-> Estimated Kaggle score ≈\", f\"{acc_tta:.4f}\",\n          \"(the final model trains on 10% more data, usually slightly higher)\")\n    del val_model\n    torch.cuda.empty_cache()\n\n# ---------------- Phase 2: retrain on all 50k from scratch ----------------\nfinal_model = train(X_all, y_all, tag=\"final 50k\")\ntorch.save(unwrap(final_model).state_dict(), os.path.join(WORK, \"resnet9_final.pt\"))\n\n# ---------------- Kaggle submission ----------------\npreds = predict(final_model, X_test, tta=True)\nsub = pd.DataFrame({\"id\": test_ids, \"label\": [CLASSES[p] for p in preds]})\nsub.to_csv(os.path.join(WORK, \"submission.csv\"), index=False)\n\nprint(sub.head())\nprint(sub[\"label\"].value_counts())\nprint(f\"Saved {os.path.join(WORK, 'submission.csv')} with {len(sub)} rows\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-09-23T05:54:21.147543Z","iopub.execute_input":"2026-09-23T05:54:21.147896Z","iopub.status.idle":"2026-09-23T06:33:17.234577Z","shell.execute_reply.started":"2026-09-23T05:54:21.147863Z","shell.execute_reply":"2026-09-23T06:33:17.233683Z"}},"outputs":[],"execution_count":null}]}