{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Subset\n\nprint(f\"PyTorch version: {torch.__version__}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-30T11:11:29.190664Z","iopub.execute_input":"2026-06-30T11:11:29.190962Z","iopub.status.idle":"2026-06-30T11:11:39.164152Z","shell.execute_reply.started":"2026-06-30T11:11:29.190913Z","shell.execute_reply":"2026-06-30T11:11:39.163367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EPOCHS              = 20       # short training run per experiment\nBATCH_SIZE          = 64\nRANDOM_SEED         = 42\n\nSUBSET_TRAIN_SIZE   = 4000    # how many of the 50,000 training images to actually use\nSUBSET_TEST_SIZE    = 1000    # how many of the 10,000 test images to actually use\nVAL_FRACTION        = 0.2     # slice of the training subset held out for validation\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ntorch.manual_seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\n\nprint(f\"Device : {DEVICE}\")\nprint(f\"Using {SUBSET_TRAIN_SIZE} train / {SUBSET_TEST_SIZE} test images \"\n      f\"out of 60,000 total in CIFAR-10\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-30T11:11:39.165472Z","iopub.execute_input":"2026-06-30T11:11:39.165913Z","iopub.status.idle":"2026-06-30T11:11:39.457142Z","shell.execute_reply.started":"2026-06-30T11:11:39.165887Z","shell.execute_reply":"2026-06-30T11:11:39.456547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.4914, 0.4822, 0.4465),   # CIFAR-10 channel mean\n                          (0.2470, 0.2435, 0.2616))   # CIFAR-10 channel std\n])\n\nfull_train_set = torchvision.datasets.CIFAR10(root=\"./data\", train=True,  download=True, transform=transform)\nfull_test_set  = torchvision.datasets.CIFAR10(root=\"./data\", train=False, download=True, transform=transform)\n\nCLASS_NAMES = full_train_set.classes\n\n# ---- Take a random slice instead of using everything ----\ntrain_pool_idx = torch.randperm(len(full_train_set))[:SUBSET_TRAIN_SIZE]\ntest_idx       = torch.randperm(len(full_test_set))[:SUBSET_TEST_SIZE]\n\nn_val     = int(SUBSET_TRAIN_SIZE * VAL_FRACTION)\nval_idx   = train_pool_idx[:n_val]\ntrain_idx = train_pool_idx[n_val:]\n\ntrain_dataset = Subset(full_train_set, train_idx)\nval_dataset   = Subset(full_train_set, val_idx)\ntest_dataset  = Subset(full_test_set,  test_idx)\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\nval_loader   = DataLoader(val_dataset,   batch_size=BATCH_SIZE, shuffle=False)\ntest_loader  = DataLoader(test_dataset,  batch_size=BATCH_SIZE, shuffle=False)\n\nprint(f\"Train: {len(train_dataset)}  |  Val: {len(val_dataset)}  |  Test: {len(test_dataset)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-30T11:11:39.457931Z","iopub.execute_input":"2026-06-30T11:11:39.458242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def denormalize(img_tensor):\n    mean = torch.tensor((0.4914, 0.4822, 0.4465)).view(3, 1, 1)\n    std  = torch.tensor((0.2470, 0.2435, 0.2616)).view(3, 1, 1)\n    return (img_tensor * std + mean).clamp(0, 1)\n\nfig, axes = plt.subplots(1, 8, figsize=(14, 2))\nfor i, ax in enumerate(axes):\n    img, label = train_dataset[i]\n    ax.imshow(denormalize(img).permute(1, 2, 0).numpy())\n    ax.set_title(CLASS_NAMES[label], fontsize=9)\n    ax.axis(\"off\")\nplt.suptitle(\"A few sample training images\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SimpleCNN(nn.Module):\n    def __init__(self, num_classes=10):\n        super().__init__()\n        self.features = nn.Sequential(\n            nn.Conv2d(3, 16, kernel_size=3, padding=1),\n            nn.BatchNorm2d(16),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2, 2),          # 32x32 -> 16x16\n\n            nn.Conv2d(16, 32, kernel_size=3, padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2, 2),          # 16x16 -> 8x8\n        )\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(32 * 8 * 8, 128),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.3),\n            nn.Linear(128, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.classifier(x)\n        return x\n\n\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\n# Quick sanity check\n_m = SimpleCNN()\nprint(f\"Model parameters : {count_parameters(_m):,}\")\nprint(f\"Output shape check: {_m(torch.randn(2, 3, 32, 32)).shape}\")  # expect [2, 10]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, criterion):\n    model.train()\n    total_loss, correct, total = 0.0, 0, 0\n\n    for images, labels in loader:\n        images, labels = images.to(DEVICE), labels.to(DEVICE)\n\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss    = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item() * images.size(0)\n        correct    += outputs.argmax(1).eq(labels).sum().item()\n        total      += labels.size(0)\n\n    return total_loss / total, 100.0 * correct / total\n\n\ndef evaluate(model, loader, criterion):\n    model.eval()\n    total_loss, correct, total = 0.0, 0, 0\n\n    with torch.no_grad():\n        for images, labels in loader:\n            images, labels = images.to(DEVICE), labels.to(DEVICE)\n            outputs = model(images)\n            loss    = criterion(outputs, labels)\n\n            total_loss += loss.item() * images.size(0)\n            correct    += outputs.argmax(1).eq(labels).sum().item()\n            total      += labels.size(0)\n\n    return total_loss / total, 100.0 * correct / total","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"experiments = [\n    {\n        \"name\"        : \"SGD + CrossEntropy\",\n        \"optimizer_fn\": lambda p: optim.SGD(p, lr=0.01, momentum=0.9),\n        \"criterion\"   : nn.CrossEntropyLoss(),\n    },\n    {\n        \"name\"        : \"Adam + CrossEntropy\",\n        \"optimizer_fn\": lambda p: optim.Adam(p, lr=0.001),\n        \"criterion\"   : nn.CrossEntropyLoss(),\n    },\n    {\n        \"name\"        : \"RMSprop + CrossEntropy\",\n        \"optimizer_fn\": lambda p: optim.RMSprop(p, lr=0.001),\n        \"criterion\"   : nn.CrossEntropyLoss(),\n    },\n    {\n        \"name\"        : \"Adam + Label Smoothing\",\n        \"optimizer_fn\": lambda p: optim.Adam(p, lr=0.001),\n        \"criterion\"   : nn.CrossEntropyLoss(label_smoothing=0.1),\n    },\n    {\n        \"name\"        : \"Adam + Hinge (MultiMargin)\",\n        \"optimizer_fn\": lambda p: optim.Adam(p, lr=0.001),\n        \"criterion\"   : nn.MultiMarginLoss(),\n    },\n]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_results = {}\n\nfor exp in experiments:\n    print(f\"\\n{'='*55}\")\n    print(f\"  {exp['name']}\")\n    print(f\"{'='*55}\")\n\n    torch.manual_seed(RANDOM_SEED)     # same starting weights for every experiment\n    model     = SimpleCNN().to(DEVICE)\n    optimizer = exp[\"optimizer_fn\"](model.parameters())\n    criterion = exp[\"criterion\"]\n\n    train_losses, val_losses = [], []\n    train_accs,   val_accs   = [], []\n\n    for epoch in range(1, EPOCHS + 1):\n        tr_loss, tr_acc   = train_one_epoch(model, train_loader, optimizer, criterion)\n        val_loss, val_acc = evaluate(model, val_loader, criterion)\n\n        train_losses.append(tr_loss); val_losses.append(val_loss)\n        train_accs.append(tr_acc);   val_accs.append(val_acc)\n\n        print(f\"  Epoch {epoch}/{EPOCHS} | \"\n              f\"Train Loss {tr_loss:.3f} Acc {tr_acc:5.1f}% | \"\n              f\"Val Loss {val_loss:.3f} Acc {val_acc:5.1f}%\")\n\n    test_loss, test_acc = evaluate(model, test_loader, criterion)\n    print(f\"  -> Test Accuracy: {test_acc:.1f}%\")\n\n    all_results[exp[\"name\"]] = {\n        \"train_losses\": train_losses, \"val_losses\": val_losses,\n        \"train_accs\"  : train_accs,   \"val_accs\"  : val_accs,\n        \"test_acc\"    : test_acc,\n    }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n = len(all_results)\nfig, axes = plt.subplots(n, 2, figsize=(11, 3.2 * n))\n\nfor i, (name, res) in enumerate(all_results.items()):\n    ep = range(1, len(res[\"train_losses\"]) + 1)\n\n    ax = axes[i, 0]\n    ax.plot(ep, res[\"train_losses\"], label=\"Train\", linewidth=2)\n    ax.plot(ep, res[\"val_losses\"],   label=\"Val\",   linewidth=2, linestyle=\"--\")\n    ax.set_title(f\"{name} — Loss\")\n    ax.set_xlabel(\"Epoch\"); ax.set_ylabel(\"Loss\")\n    ax.legend(); ax.grid(alpha=0.3)\n\n    ax = axes[i, 1]\n    ax.plot(ep, res[\"train_accs\"], label=\"Train\", linewidth=2)\n    ax.plot(ep, res[\"val_accs\"],   label=\"Val\",   linewidth=2, linestyle=\"--\")\n    ax.set_title(f\"{name} — Accuracy  (Test: {res['test_acc']:.1f}%)\")\n    ax.set_xlabel(\"Epoch\"); ax.set_ylabel(\"Accuracy (%)\")\n    ax.legend(); ax.grid(alpha=0.3)\n\nplt.tight_layout()\nplt.savefig(\"optimizer_loss_curves.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"names      = list(all_results.keys())\nval_final  = [all_results[n][\"val_accs\"][-1] for n in names]\ntest_final = [all_results[n][\"test_acc\"]     for n in names]\n\nx, width = np.arange(len(names)), 0.35\n\nfig, ax = plt.subplots(figsize=(10, 4.5))\nax.bar(x - width/2, val_final,  width, label=\"Final Val Acc\", alpha=0.85)\nax.bar(x + width/2, test_final, width, label=\"Test Acc\",      alpha=0.85)\n\nax.set_xticks(x)\nax.set_xticklabels(names, rotation=15, ha=\"right\")\nax.set_ylabel(\"Accuracy (%)\")\nax.set_title(\"Validation vs Test Accuracy — All Experiments\")\nax.legend(); ax.grid(alpha=0.3, axis=\"y\")\n\nplt.tight_layout()\nplt.savefig(\"summary_bar.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"{'Experiment':<32}{'Final Val Acc':>15}{'Test Acc':>12}\")\nprint(\"-\" * 59)\nfor name, res in all_results.items():\n    print(f\"{name:<32}{res['val_accs'][-1]:>14.1f}%{res['test_acc']:>11.1f}%\")\n\nbest = max(all_results, key=lambda k: all_results[k][\"test_acc\"])\nprint(f\"\\nBest by test accuracy: {best}  ({all_results[best]['test_acc']:.1f}%)\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}