{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"24c968bd-51b7-47fe-93d7-02af23acca38","cell_type":"markdown","source":"# CIFAR-10 from scratch: ResNet-18 (randomly initialized, no pre-trained weights)\n\nPlan\n1. Train on the official CIFAR-10 training set (same as the Kaggle training set).\n2. Evaluate on the official CIFAR-10 test set to estimate the Kaggle score.\n3. Predict on the Kaggle test set (300,000 images) and write `submission.csv`.\n\nNotebook settings: **Accelerator = GPU** and **Internet = On**\n(Internet is needed for torchvision to download the official CIFAR-10 files\nand for `pip install py7zr` if it is not already installed.)","metadata":{}},{"id":"fa52b9aa-78fd-4e5b-9d3d-e2d97dc3c546","cell_type":"code","source":"import os, glob, time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as T\nfrom torch.utils.data import DataLoader, Dataset\nfrom PIL import Image\n\nSEED = 0\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nuse_amp = device == \"cuda\"\ntorch.backends.cudnn.benchmark = True\nprint(\"device:\", device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:34:32.277345Z","iopub.execute_input":"2026-09-19T15:34:32.277597Z","iopub.status.idle":"2026-09-19T15:34:43.514157Z","shell.execute_reply.started":"2026-09-19T15:34:32.277572Z","shell.execute_reply":"2026-09-19T15:34:43.513113Z"}},"outputs":[],"execution_count":null},{"id":"be391de7-f5b0-4277-bd98-88a2cec29c03","cell_type":"markdown","source":"## 1. Data (official CIFAR-10) + augmentation","metadata":{}},{"id":"3af8f2e0-8711-449b-81e0-30402cefdcc8","cell_type":"code","source":"MEAN = (0.4914, 0.4822, 0.4465)\nSTD = (0.2470, 0.2435, 0.2616)\n\ntrain_tf = T.Compose([\n    T.RandomCrop(32, padding=4, padding_mode=\"reflect\"),\n    T.RandomHorizontalFlip(),\n    T.ToTensor(),\n    T.Normalize(MEAN, STD),\n])\ntest_tf = T.Compose([T.ToTensor(), T.Normalize(MEAN, STD)])\n\ntrain_set = torchvision.datasets.CIFAR10(\"./data\", train=True, download=True, transform=train_tf)\ntest_set = torchvision.datasets.CIFAR10(\"./data\", train=False, download=True, transform=test_tf)\nclasses = train_set.classes  # ['airplane', 'automobile', 'bird', ..., 'truck']\nprint(len(train_set), len(test_set), classes)\n\nBATCH_SIZE = 128\nNUM_WORKERS = 4\ntrain_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True,\n                          num_workers=NUM_WORKERS, pin_memory=True,\n                          drop_last=True, persistent_workers=True)\ntest_loader = DataLoader(test_set, batch_size=512, shuffle=False,\n                         num_workers=NUM_WORKERS, pin_memory=True,\n                         persistent_workers=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:34:43.516143Z","iopub.execute_input":"2026-09-19T15:34:43.516619Z","iopub.status.idle":"2026-09-19T15:35:02.803311Z","shell.execute_reply.started":"2026-09-19T15:34:43.516592Z","shell.execute_reply":"2026-09-19T15:35:02.802355Z"}},"outputs":[],"execution_count":null},{"id":"51b5acc6-f19e-4d10-8f2c-eb0c484c76ec","cell_type":"markdown","source":"## 2. Model: ResNet-18 (CIFAR variant), random initialization","metadata":{}},{"id":"715847c7-f2a9-49ca-a56a-2e3d0e80d8d2","cell_type":"code","source":"class BasicBlock(nn.Module):\n    def __init__(self, in_ch, out_ch, stride=1):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_ch, out_ch, 3, stride, 1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_ch)\n        self.conv2 = nn.Conv2d(out_ch, out_ch, 3, 1, 1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_ch)\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_ch != out_ch:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_ch, out_ch, 1, stride, bias=False),\n                nn.BatchNorm2d(out_ch),\n            )\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        return F.relu(out + self.shortcut(x))\n\n\nclass ResNet18(nn.Module):\n    def __init__(self, num_classes=10):\n        super().__init__()\n        # 3x3 stem, no max-pool: better suited to 32x32 images\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, 64, 3, 1, 1, bias=False), nn.BatchNorm2d(64), nn.ReLU(inplace=True)\n        )\n        cfg = [(64, 1), (128, 2), (256, 2), (512, 2)]\n        layers, in_ch = [], 64\n        for out_ch, stride in cfg:\n            layers += [BasicBlock(in_ch, out_ch, stride), BasicBlock(out_ch, out_ch, 1)]\n            in_ch = out_ch\n        self.layers = nn.Sequential(*layers)\n        self.fc = nn.Linear(512, num_classes)\n\n        # explicit random initialization (no pre-trained weights anywhere)\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n\n    def forward(self, x):\n        x = self.layers(self.stem(x))\n        x = F.adaptive_avg_pool2d(x, 1).flatten(1)\n        return self.fc(x)\n\n\nmodel = ResNet18().to(device).to(memory_format=torch.channels_last)\nprint(f\"parameters: {sum(p.numel() for p in model.parameters()) / 1e6:.2f}M\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:35:02.804490Z","iopub.execute_input":"2026-09-19T15:35:02.805356Z","iopub.status.idle":"2026-09-19T15:35:03.344881Z","shell.execute_reply.started":"2026-09-19T15:35:02.805328Z","shell.execute_reply":"2026-09-19T15:35:03.344178Z"}},"outputs":[],"execution_count":null},{"id":"dfff9dcc-9318-4473-ae89-b3aabae9e0ae","cell_type":"markdown","source":"## 3. Training","metadata":{}},{"id":"50159a33-4a01-4b73-9d90-9d38bc1ab27c","cell_type":"code","source":"EPOCHS = 30\nMAX_LR = 0.1\nWEIGHT_DECAY = 5e-4\n\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1)\noptimizer = torch.optim.SGD(model.parameters(), lr=MAX_LR, momentum=0.9,\n                            weight_decay=WEIGHT_DECAY, nesterov=True)\nscheduler = torch.optim.lr_scheduler.OneCycleLR(\n    optimizer, max_lr=MAX_LR, epochs=EPOCHS, steps_per_epoch=len(train_loader), pct_start=0.2\n)\nscaler = torch.amp.GradScaler(\"cuda\", enabled=use_amp)\n\n\n@torch.no_grad()\ndef evaluate(loader):\n    model.eval()\n    correct, total, loss_sum = 0, 0, 0.0\n    for x, y in loader:\n        x = x.to(device, non_blocking=True).to(memory_format=torch.channels_last)\n        y = y.to(device, non_blocking=True)\n        with torch.autocast(device_type=\"cuda\", dtype=torch.float16, enabled=use_amp):\n            out = model(x)\n            loss_sum += F.cross_entropy(out, y, reduction=\"sum\").item()\n        correct += (out.argmax(1) == y).sum().item()\n        total += y.numel()\n    return loss_sum / total, correct / total\n\n\nhistory = []\nfor epoch in range(1, EPOCHS + 1):\n    model.train()\n    t0, run_loss, run_correct, seen = time.time(), 0.0, 0, 0\n    for x, y in train_loader:\n        x = x.to(device, non_blocking=True).to(memory_format=torch.channels_last)\n        y = y.to(device, non_blocking=True)\n        optimizer.zero_grad(set_to_none=True)\n        with torch.autocast(device_type=\"cuda\", dtype=torch.float16, enabled=use_amp):\n            out = model(x)\n            loss = criterion(out, y)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()\n\n        run_loss += loss.item() * y.size(0)\n        run_correct += (out.argmax(1) == y).sum().item()\n        seen += y.size(0)\n\n    test_loss, test_acc = evaluate(test_loader)\n    history.append((epoch, run_loss / seen, run_correct / seen, test_loss, test_acc))\n    print(f\"epoch {epoch:02d}/{EPOCHS} | train loss {run_loss/seen:.3f} acc {run_correct/seen:.4f} \"\n          f\"| test loss {test_loss:.3f} acc {test_acc:.4f} | {time.time()-t0:.0f}s\")\n\ntorch.save(model.state_dict(), \"resnet18_cifar10_scratch.pt\")\nfinal_loss, final_acc = evaluate(test_loader)\nprint(f\"\\nFinal accuracy on the official CIFAR-10 test set: {final_acc*100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:35:03.345900Z","iopub.execute_input":"2026-09-19T15:35:03.346279Z","iopub.status.idle":"2026-09-19T15:43:46.895385Z","shell.execute_reply.started":"2026-09-19T15:35:03.346227Z","shell.execute_reply":"2026-09-19T15:43:46.894241Z"}},"outputs":[],"execution_count":null},{"id":"e3a8131a-ffad-4274-a6f7-713e876b3ccf","cell_type":"markdown","source":"## 4. Learning curves (useful for your report)","metadata":{}},{"id":"ca238505-5844-40f2-9e9c-2831715c1fc5","cell_type":"code","source":"import matplotlib.pyplot as plt\n\nh = np.array(history)\nfig, ax = plt.subplots(1, 2, figsize=(11, 4))\nax[0].plot(h[:, 0], h[:, 1], label=\"train\"); ax[0].plot(h[:, 0], h[:, 3], label=\"test\")\nax[0].set_title(\"Loss\"); ax[0].set_xlabel(\"epoch\"); ax[0].legend()\nax[1].plot(h[:, 0], h[:, 2], label=\"train\"); ax[1].plot(h[:, 0], h[:, 4], label=\"test\")\nax[1].set_title(\"Accuracy\"); ax[1].set_xlabel(\"epoch\"); ax[1].legend()\nplt.tight_layout(); plt.savefig(\"curves.png\", dpi=120); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:43:46.897582Z","iopub.execute_input":"2026-09-19T15:43:46.898160Z","iopub.status.idle":"2026-09-19T15:43:47.575449Z","shell.execute_reply.started":"2026-09-19T15:43:46.898111Z","shell.execute_reply":"2026-09-19T15:43:47.574400Z"}},"outputs":[],"execution_count":null},{"id":"8be9e596-6885-49ca-bd74-79a6c8b6aa01","cell_type":"markdown","source":"## 5. Predict on the Kaggle test set and build `submission.csv`\n\nAdd the competition data to the notebook (**Add Input -> Competitions -> CIFAR-10 - Object Recognition in Images**).\nThe Kaggle test set ships as `test.7z` (300,000 PNGs named `1.png ... 300000.png`).","metadata":{}},{"id":"d44f9727-c522-48c1-a290-774bd4e5baaa","cell_type":"code","source":"try:\n    import py7zr\nexcept ImportError:\n    os.system(\"pip install -q py7zr\")\n    import py7zr\n\narchives = glob.glob(\"/kaggle/input/**/test.7z\", recursive=True)\nassert archives, \"test.7z not found - did you add the CIFAR-10 competition as an input?\"\nsample_sub_path = glob.glob(\"/kaggle/input/**/sampleSubmission.csv\", recursive=True)[0]\n\nEXTRACT_DIR = \"/kaggle/working/kaggle_test\"\nif not os.path.isdir(EXTRACT_DIR):\n    with py7zr.SevenZipFile(archives[0], mode=\"r\") as z:\n        z.extractall(EXTRACT_DIR)\n\nfiles = glob.glob(f\"{EXTRACT_DIR}/**/*.png\", recursive=True)\nprint(\"test images:\", len(files))\n\n\nclass KaggleTestSet(Dataset):\n    def __init__(self, files):\n        self.files = files\n\n    def __len__(self):\n        return len(self.files)\n\n    def __getitem__(self, i):\n        img = Image.open(self.files[i]).convert(\"RGB\")\n        return test_tf(img), int(os.path.splitext(os.path.basename(self.files[i]))[0])\n\n\nkaggle_loader = DataLoader(KaggleTestSet(files), batch_size=512, shuffle=False,\n                           num_workers=NUM_WORKERS, pin_memory=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:43:47.576696Z","iopub.execute_input":"2026-09-19T15:43:47.577342Z","iopub.status.idle":"2026-09-19T15:47:58.002472Z","shell.execute_reply.started":"2026-09-19T15:43:47.577281Z","shell.execute_reply":"2026-09-19T15:47:58.001478Z"}},"outputs":[],"execution_count":null},{"id":"601fcbb3-3b24-4bcf-a174-5995f0cbddb3","cell_type":"code","source":"model.eval()\nall_ids, all_preds = [], []\nwith torch.no_grad():\n    for x, ids in kaggle_loader:\n        x = x.to(device, non_blocking=True).to(memory_format=torch.channels_last)\n        with torch.autocast(device_type=\"cuda\", dtype=torch.float16, enabled=use_amp):\n            logits = model(x) + model(torch.flip(x, dims=[3]))  # horizontal-flip TTA\n        all_preds.append(logits.argmax(1).cpu())\n        all_ids.append(ids)\n\npreds = torch.cat(all_preds).numpy()\nids = torch.cat(all_ids).numpy()\n\nsubmission = pd.DataFrame({\"id\": ids, \"label\": [classes[p] for p in preds]}).sort_values(\"id\")\n\n# make sure the file matches Kaggle's expected format exactly\nsample = pd.read_csv(sample_sub_path)\nassert list(submission.columns) == list(sample.columns)\nassert len(submission) == len(sample) and (submission[\"id\"].values == sample[\"id\"].sort_values().values).all()\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(submission.head())\nprint(submission[\"label\"].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-19T15:47:58.004858Z","iopub.execute_input":"2026-09-19T15:47:58.005246Z","iopub.status.idle":"2026-09-19T15:49:07.677625Z","shell.execute_reply.started":"2026-09-19T15:47:58.005216Z","shell.execute_reply":"2026-09-19T15:49:07.676647Z"}},"outputs":[],"execution_count":null}]}