{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"2e57db9a","cell_type":"markdown","source":"# How this Implementation Achieves Over 81% Accuracy?\n\n**Key update:** the training recipe was upgraded from a basic setup to a modern CIFAR-10 pipeline (augmentation + normalization + SGD with momentum/weight decay + warmup/cosine LR schedule). This is the biggest driver of better final accuracy and training stability.\n\n## 1) Training Recipe Updates\n- Added **data augmentation** in training: `RandomHorizontalFlip` and `RandomCrop(32, 4)`.\n- Added **input normalization** with CIFAR-10 mean/std.\n- Switched optimizer from **Adam (lr=1e-3)** to **SGD (lr=5e-2, momentum=0.9, weight_decay=1e-4)**.\n- Added **per-step LR schedule**: linear warmup (`WARMUP_EPOCHS`) then cosine decay.\n- Added both **train accuracy and test accuracy** evaluation every epoch (better overfitting/underfitting visibility).\n\n## 2) Engineering Updates\n- Moved tunable knobs to a dedicated hyperparameter cell: `BATCH_SIZE`, `LEARNING_RATE`, `WEIGHT_DECAY`, `WARMUP_EPOCHS`, `RANDOM_SEED`.\n- Added reproducibility controls: Python/NumPy/Torch seeds + deterministic mode.\n- Improved data pipeline/runtime setup: `num_workers=4`, `pin_memory=True`, `non_blocking=True`.\n- Improved observability with `tqdm` progress bar showing live loss and LR.\n- Cleaner evaluation path with `@torch.inference_mode()` and type hints in function/model signatures.","metadata":{}},{"id":"d1b6d379","cell_type":"code","source":"# Hyper-parameters\nBATCH_SIZE = 64\nLEARNING_RATE = 5e-2\nWEIGHT_DECAY = 1e-4\nWARMUP_EPOCHS = 1\nRANDOM_SEED = 3407 # torch.manual_seed(3407) is all you need! [doge]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-14T14:42:24.857170Z","iopub.execute_input":"2026-04-14T14:42:24.857497Z","iopub.status.idle":"2026-04-14T14:42:24.866899Z","shell.execute_reply.started":"2026-04-14T14:42:24.857464Z","shell.execute_reply":"2026-04-14T14:42:24.865976Z"}},"outputs":[],"execution_count":null},{"id":"8da9b4e2","cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn as nn\nfrom typing import Optional, Callable\nfrom torchvision.transforms import v2 as transforms\nfrom tqdm import tqdm\n\n# Configure Reproducibility Settings\nimport random\nimport numpy as np\n\n\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\ntorch.manual_seed(RANDOM_SEED)\ntorch.use_deterministic_algorithms(True)\ntorch.backends.cudnn.benchmark = False\n\n# Device configuration\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n## You cannot change the parameters inside the CNN ##\n\n# 3x3 convolution\ndef conv3x3(in_channels: int, out_channels: int, stride: int = 1):\n    return nn.Conv2d(\n        in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False\n    )\n\n\n# Residual block\nclass ResidualBlock(nn.Module):\n    def __init__(\n        self,\n        in_channels: int,\n        out_channels: int,\n        stride: int = 1,\n        downsample: Optional[Callable[..., torch.Tensor]] = None,\n    ):\n        super(ResidualBlock, self).__init__()\n        self.conv1 = conv3x3(in_channels, out_channels, stride)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(out_channels, out_channels)\n        self.downsample = downsample\n\n    def forward(self, x: torch.Tensor):\n        residual = x\n        out = self.conv1(x)\n        out = self.relu(out)\n        out = self.conv2(out)\n        if self.downsample:\n            residual = self.downsample(x)\n        out += residual\n        out = self.relu(out)\n        return out\n\n\n# ResNet\nclass ResNet(nn.Module):\n    def __init__(\n        self, block: type[ResidualBlock], layers: list[int], num_classes: int = 10\n    ):\n        super(ResNet, self).__init__()\n        self.in_channels = 16\n        self.conv = conv3x3(3, 16)\n        self.relu = nn.ReLU(inplace=True)\n        self.layer1 = self.make_layer(block, 16, layers[0])\n        self.layer2 = self.make_layer(block, 32, layers[1], 2)\n        self.layer3 = self.make_layer(block, 64, layers[2], 2)\n        self.avg_pool = nn.AvgPool2d(8)\n        self.fc = nn.Linear(64, num_classes)\n\n    def make_layer(\n        self,\n        block: type[ResidualBlock],\n        out_channels: int,\n        blocks: int,\n        stride: int = 1,\n    ):\n        downsample = None\n        if (stride != 1) or (self.in_channels != out_channels):\n            downsample = nn.Sequential(\n                conv3x3(self.in_channels, out_channels, stride=stride),\n                nn.BatchNorm2d(out_channels),\n            )\n        layers: list[ResidualBlock] = []\n        layers.append(block(self.in_channels, out_channels, stride, downsample))\n        self.in_channels = out_channels\n        for _ in range(1, blocks):\n            layers.append(block(out_channels, out_channels))\n        return nn.Sequential(*layers)\n\n    def forward(self, x: torch.Tensor):\n        out: torch.Tensor = self.conv(x)\n        out = self.relu(out)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.avg_pool(out)\n        out = out.view(out.size(0), -1)\n        out = self.fc(out)\n        return out\n\n\n#####################################################################################\n########################## You cannot check what's below ############################\n\nnum_epochs = 15  # this cannot change!\nmodel = ResNet(ResidualBlock, [1, 1, 1]).to(device)\n\n########################## You cannot check what's above ############################\n#####################################################################################\n# Image preprocessing modules\n# transform = transforms.Compose([transforms.ToTensor()])\nmean = (0.49139967861519607843, 0.48215840839460784314, 0.44653091444546568627)\nstd = (0.21117028181572183225, 0.20857934290628859220, 0.21205155387102001073)\n\ntrain_transform = transforms.Compose(\n    [\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomCrop(32, 4),\n        transforms.ToImage(),\n        transforms.ToDtype(torch.float32, scale=True),\n        transforms.Normalize(mean, std, inplace=True),\n        transforms.ToPureTensor(),\n    ]\n)\ntest_transform = transforms.Compose(\n    [\n        transforms.ToImage(),\n        transforms.ToDtype(torch.float32, scale=True),\n        transforms.Normalize(mean, std, inplace=True),\n        transforms.ToPureTensor(),\n    ]\n)\n\n# CIFAR-10 dataset\ntrain_dataset = torchvision.datasets.CIFAR10(\n    root=\"./data\", train=True, transform=train_transform, download=True\n)\n\ntest_dataset = torchvision.datasets.CIFAR10(\n    root=\"./data\", train=False, transform=test_transform, download=True\n)\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(\n    dataset=train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    num_workers=4,\n    pin_memory=True,\n)\n\ntest_loader = torch.utils.data.DataLoader(\n    dataset=test_dataset,\n    batch_size=BATCH_SIZE * 4,\n    shuffle=False,\n    num_workers=4,\n    pin_memory=True,\n)\n# Loss, optimizer, and LR scheduler\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(\n    model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY, momentum=0.9\n)\n\n# Per-step LR schedule: linear warmup, then cosine decay\nsteps_per_epoch = len(train_loader)\ntotal_steps = num_epochs * steps_per_epoch\nwarmup_steps = WARMUP_EPOCHS * steps_per_epoch\n\nwarmup_scheduler = torch.optim.lr_scheduler.LinearLR(\n    optimizer, start_factor=0.1, end_factor=1.0, total_iters=warmup_steps\n)\ncosine_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n    optimizer, T_max=max(total_steps - warmup_steps, 1), eta_min=1e-4\n)\nscheduler = torch.optim.lr_scheduler.SequentialLR(\n    optimizer,\n    schedulers=[warmup_scheduler, cosine_scheduler],\n    milestones=[warmup_steps],\n)\n\n\n@torch.inference_mode()\ndef evaluate(model: nn.Module, data_loader: torch.utils.data.DataLoader):\n    model.eval()\n    correct = 0\n    total = 0\n    for images, labels in data_loader:\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n    accuracy = 100 * correct / total\n    return accuracy\n\n\n# Train the model\nfor epoch in range(num_epochs):\n    model.train()\n    for images, labels in (progress_bar := tqdm(train_loader)):\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n\n        # Forward pass\n        outputs: torch.Tensor = model(images)\n        loss: torch.Tensor = criterion(outputs, labels)\n\n        # Backward and optimize\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n\n        current_lr = optimizer.param_groups[0][\"lr\"]\n        progress_bar.set_description(\n            f\"Epoch {epoch + 1}/{num_epochs}, Loss: {loss.item():.4f}, LR: {current_lr:.6f}\"\n        )\n\n    # Test the model\n    train_acc = evaluate(model, train_loader)\n    test_acc = evaluate(model, test_loader)\n    print(\n        f\"[{epoch + 1}/{num_epochs}] Train Acc: {train_acc:.2f}%, Test Acc: {test_acc:.2f}%\"\n    )\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), \"resnet.ckpt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-14T14:42:24.868349Z","iopub.execute_input":"2026-04-14T14:42:24.868819Z","iopub.status.idle":"2026-04-14T14:48:48.390575Z","shell.execute_reply.started":"2026-04-14T14:42:24.868796Z","shell.execute_reply":"2026-04-14T14:48:48.389658Z"}},"outputs":[],"execution_count":null}]}