{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =====================\n# Cell 0: Environment\n# =====================\n\nimport os, math, random, time, glob, json\nimport numpy as np\nfrom pathlib import Path\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torch.optim import SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision import datasets, transforms, models\n\nfrom PIL import Image\n\nSEED = 123\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED)\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device:', DEVICE)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:47:09.387754Z","iopub.execute_input":"2025-09-23T21:47:09.388579Z","iopub.status.idle":"2025-09-23T21:47:14.041281Z","shell.execute_reply.started":"2025-09-23T21:47:09.388546Z","shell.execute_reply":"2025-09-23T21:47:14.040663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 1: Config\n# =====================\nclass CFG:\n    # Data & training\n    img_size = 32\n    epochs = 120            # Increase if you have more time (e.g., 200)\n    batch_size = 128\n    num_workers = 2\n\n    # Optimizer / LR\n    lr = 0.1\n    weight_decay = 5e-4\n    momentum = 0.9\n    warmup_epochs = 5\n\n    # Regularization\n    label_smoothing = 0.1\n    mixup_alpha = 0.2\n    cutmix_alpha = 1.0      # set to 0 to disable CutMix\n    mixup_prob = 0.7        # probability to apply Mixup/CutMix per batch\n\n    # EMA\n    use_ema = True\n    ema_decay = 0.999\n       \n    # Paths\n    work_dir = '.'\n    submission_path = os.path.join(work_dir, 'submission.csv')\n\n    # Classes for CIFAR-10 (match Kaggle label strings)\n    classes = ['airplane', 'automobile', 'bird', 'cat', 'deer',\n               'dog', 'frog', 'horse', 'ship', 'truck']\n\ncfg = CFG()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:47:24.874830Z","iopub.execute_input":"2025-09-23T21:47:24.875437Z","iopub.status.idle":"2025-09-23T21:47:24.880080Z","shell.execute_reply.started":"2025-09-23T21:47:24.875413Z","shell.execute_reply":"2025-09-23T21:47:24.879528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 2: Transforms\n# =====================\n\nCIFAR10_MEAN = (0.4914, 0.4822, 0.4465)\nCIFAR10_STD  = (0.2023, 0.1994, 0.2010)\n\ntrain_transform = transforms.Compose([\n    transforms.RandomCrop(cfg.img_size, padding=4),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandAugment(num_ops=2, magnitude=9),  # randaugment is available in torchvision >= 0.13\n    transforms.ToTensor(),\n    transforms.Normalize(CIFAR10_MEAN, CIFAR10_STD),\n])\n\nvalid_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(CIFAR10_MEAN, CIFAR10_STD),\n])","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:47:32.046842Z","iopub.execute_input":"2025-09-23T21:47:32.047430Z","iopub.status.idle":"2025-09-23T21:47:32.052255Z","shell.execute_reply.started":"2025-09-23T21:47:32.047403Z","shell.execute_reply":"2025-09-23T21:47:32.051526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 3: Data\n# =====================\n\ndata_root = './data'\ntrain_set = datasets.CIFAR10(root=data_root, train=True,  download=True, transform=train_transform)\nvalid_set = datasets.CIFAR10(root=data_root, train=False, download=True, transform=valid_transform)\n\ntrain_loader = DataLoader(train_set, batch_size=cfg.batch_size, shuffle=True,\n                          num_workers=cfg.num_workers, pin_memory=True)\nvalid_loader = DataLoader(valid_set, batch_size=cfg.batch_size, shuffle=False,\n                          num_workers=cfg.num_workers, pin_memory=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:47:37.535497Z","iopub.execute_input":"2025-09-23T21:47:37.535789Z","iopub.status.idle":"2025-09-23T21:47:45.886717Z","shell.execute_reply.started":"2025-09-23T21:47:37.535767Z","shell.execute_reply":"2025-09-23T21:47:45.885968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 4: Model (ResNet-18) \n# =====================\n\nclass ResNet18Scratch(nn.Module):\n    def __init__(self, num_classes=10):\n        super().__init__()\n        m = models.resnet18(weights=None)  # ensures random init\n        m.fc = nn.Linear(m.fc.in_features, num_classes)\n        self.model = m\n        self._init_weights()\n\n    def _init_weights(self):\n        # He (Kaiming) init for convs, zero bias; normal for linear\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n                elif isinstance(m, nn.BatchNorm2d):\n                    nn.init.ones_(m.weight)\n                    nn.init.zeros_(m.bias)\n                elif isinstance(m, nn.Linear):\n                    nn.init.normal_(m.weight, 0, 0.01)\n                    nn.init.zeros_(m.bias)\n\n    def forward(self, x):\n        return self.model(x)\n\nmodel = ResNet18Scratch(num_classes=10).to(DEVICE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:48:53.536615Z","iopub.execute_input":"2025-09-23T21:48:53.537456Z","iopub.status.idle":"2025-09-23T21:48:53.927559Z","shell.execute_reply.started":"2025-09-23T21:48:53.537430Z","shell.execute_reply":"2025-09-23T21:48:53.926794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 5: Losses (Label smoothing), MixUp/CutMix, EMA\n# =====================\nclass LabelSmoothingCE(nn.Module):\n    def __init__(self, smoothing=0.1):\n        super().__init__()\n        self.smoothing = smoothing\n\n    def forward(self, pred, target):\n        # pred: (B, C), logits; target: (B,) int\n        log_probs = F.log_softmax(pred, dim=-1)\n        n_classes = pred.size(-1)\n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (n_classes - 1))\n            true_dist.scatter_(1, target.unsqueeze(1), 1.0 - self.smoothing)\n        return torch.mean(torch.sum(-true_dist * log_probs, dim=-1))\n\ncriterion = LabelSmoothingCE(cfg.label_smoothing)\n\n# MixUp & CutMix utilities\n\ndef rand_bbox(W, H, lam):\n    # for CutMix; returns bbox coords\n    cut_rat = math.sqrt(1.0 - lam)\n    cut_w, cut_h = int(W * cut_rat), int(H * cut_rat)\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    x1 = np.clip(cx - cut_w // 2, 0, W)\n    y1 = np.clip(cy - cut_h // 2, 0, H)\n    x2 = np.clip(cx + cut_w // 2, 0, W)\n    y2 = np.clip(cy + cut_h // 2, 0, H)\n    return x1, y1, x2, y2\n\n@torch.no_grad()\ndef update_ema(ema_model, model, decay):\n    msd = model.state_dict()\n    for k, v in ema_model.state_dict().items():\n        if k in msd:\n            ema_model.state_dict()[k].copy_(decay * v + (1 - decay) * msd[k])\n\nema_model = None\nif cfg.use_ema:\n    ema_model = ResNet18Scratch(num_classes=10).to(DEVICE)\n    ema_model.load_state_dict(model.state_dict())\n    for p in ema_model.parameters():\n        p.requires_grad_(False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:48:57.843571Z","iopub.execute_input":"2025-09-23T21:48:57.843830Z","iopub.status.idle":"2025-09-23T21:48:58.082904Z","shell.execute_reply.started":"2025-09-23T21:48:57.843808Z","shell.execute_reply":"2025-09-23T21:48:58.082363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 6: Optimizer & Scheduler with Warmup\n# =====================\noptimizer = SGD(model.parameters(), lr=cfg.lr, momentum=cfg.momentum, weight_decay=cfg.weight_decay)\nscheduler = CosineAnnealingLR(optimizer, T_max=cfg.epochs - cfg.warmup_epochs)\n\nscaler = torch.amp.GradScaler(enabled=(DEVICE.type == 'cuda'))\n\n# Warmup helper\nclass WarmupScheduler:\n    def __init__(self, optimizer, warmup_epochs, base_lr):\n        self.optimizer = optimizer\n        self.warmup_epochs = warmup_epochs\n        self.base_lr = base_lr\n        self.current_epoch = 0\n    def step(self):\n        self.current_epoch += 1\n    def set_epoch_lr(self, epoch):\n        if epoch < self.warmup_epochs:\n            lr = self.base_lr * (epoch + 1) / self.warmup_epochs\n            for pg in self.optimizer.param_groups:\n                pg['lr'] = lr\n\nwarmup = WarmupScheduler(optimizer, cfg.warmup_epochs, cfg.lr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:51:06.886728Z","iopub.execute_input":"2025-09-23T21:51:06.887465Z","iopub.status.idle":"2025-09-23T21:51:06.893263Z","shell.execute_reply.started":"2025-09-23T21:51:06.887438Z","shell.execute_reply":"2025-09-23T21:51:06.892602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 7: Train & Validate loops\n# =====================\n\ndef train_one_epoch(epoch):\n    model.train()\n    running_loss, running_correct, total = 0.0, 0, 0\n\n    for images, targets in train_loader:\n        images, targets = images.to(DEVICE), targets.to(DEVICE)\n\n        # Decide on MixUp/CutMix\n        use_mix = random.random() < cfg.mixup_prob\n        if use_mix:\n            r = random.random()\n            if r < 0.5 and cfg.mixup_alpha > 0:  # MixUp\n                lam = np.random.beta(cfg.mixup_alpha, cfg.mixup_alpha)\n                idx = torch.randperm(images.size(0)).to(DEVICE)\n                mixed = lam * images + (1 - lam) * images[idx]\n                targets_a, targets_b = targets, targets[idx]\n                with torch.cuda.amp.autocast(enabled=(DEVICE.type=='cuda')):\n                    logits = model(mixed)\n                    loss = lam * criterion(logits, targets_a) + (1 - lam) * criterion(logits, targets_b)\n            elif cfg.cutmix_alpha > 0:            # CutMix\n                lam = np.random.beta(cfg.cutmix_alpha, cfg.cutmix_alpha)\n                idx = torch.randperm(images.size(0)).to(DEVICE)\n                x1, y1, x2, y2 = rand_bbox(images.size(3), images.size(2), lam)\n                images2 = images[idx].clone()\n                images[:, :, y1:y2, x1:x2] = images2[:, :, y1:y2, x1:x2]\n                lam = 1 - ((x2 - x1) * (y2 - y1) / (images.size(-1) * images.size(-2)))\n                targets_a, targets_b = targets, targets[idx]\n                with torch.cuda.amp.autocast(enabled=(DEVICE.type=='cuda')):\n                    logits = model(images)\n                    loss = lam * criterion(logits, targets_a) + (1 - lam) * criterion(logits, targets_b)\n            else:\n                with torch.cuda.amp.autocast(enabled=(DEVICE.type=='cuda')):\n                    logits = model(images)\n                    loss = criterion(logits, targets)\n        else:\n            with torch.cuda.amp.autocast(enabled=(DEVICE.type=='cuda')):\n                logits = model(images)\n                loss = criterion(logits, targets)\n\n        optimizer.zero_grad(set_to_none=True)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        if cfg.use_ema:\n            update_ema(ema_model, model, cfg.ema_decay)\n\n        # Metrics\n        with torch.no_grad():\n            preds = logits.argmax(1)\n            running_correct += (preds == targets).sum().item()\n            total += targets.size(0)\n            running_loss += loss.item() * targets.size(0)\n\n    # Scheduler step\n    if epoch < cfg.warmup_epochs:\n        warmup.set_epoch_lr(epoch)\n    else:\n        scheduler.step()\n\n    train_acc = running_correct / total\n    train_loss = running_loss / total\n    return train_loss, train_acc\n\n\n@torch.no_grad()\ndef evaluate(use_ema=True):\n    net = ema_model if (cfg.use_ema and use_ema) else model\n    net.eval()\n    total, correct, running_loss = 0, 0, 0.0\n\n    for images, targets in valid_loader:\n        images, targets = images.to(DEVICE), targets.to(DEVICE)\n        with torch.cuda.amp.autocast(enabled=(DEVICE.type=='cuda')):\n            logits = net(images)\n            loss = criterion(logits, targets)\n        preds = logits.argmax(1)\n        correct += (preds == targets).sum().item()\n        total += targets.size(0)\n        running_loss += loss.item() * targets.size(0)\n\n    return running_loss/total, correct/total","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:51:11.290514Z","iopub.execute_input":"2025-09-23T21:51:11.291271Z","iopub.status.idle":"2025-09-23T21:51:11.304644Z","shell.execute_reply.started":"2025-09-23T21:51:11.291223Z","shell.execute_reply":"2025-09-23T21:51:11.303877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 8: Training loop\n# =====================\nbest_acc = 0.0\nbest_path = os.path.join(cfg.work_dir, 'best_model.pth')\n\nfor epoch in range(cfg.epochs):\n    t0 = time.time()\n    tr_loss, tr_acc = train_one_epoch(epoch)\n    val_loss, val_acc = evaluate(use_ema=True)\n    dt = time.time() - t0\n\n    if val_acc > best_acc:\n        best_acc = val_acc\n        torch.save((ema_model if cfg.use_ema else model).state_dict(), best_path)\n\n    # Print progress\n    current_lr = optimizer.param_groups[0]['lr']\n    print(f\"Epoch {epoch+1:03d}/{cfg.epochs} | lr {current_lr:.5f} | \"\n          f\"train_loss {tr_loss:.4f} acc {tr_acc:.4f} | val_loss {val_loss:.4f} acc {val_acc:.4f} | {dt:.1f}s\")\n\nprint('Best Val Acc:', best_acc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T21:51:18.006026Z","iopub.execute_input":"2025-09-23T21:51:18.006325Z","iopub.status.idle":"2025-09-23T22:53:12.311542Z","shell.execute_reply.started":"2025-09-23T21:51:18.006302Z","shell.execute_reply":"2025-09-23T22:53:12.310615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 9: Load best model and evaluate (final estimate of Kaggle score)\n# =====================\nif os.path.exists(best_path):\n    if cfg.use_ema:\n        ema_model.load_state_dict(torch.load(best_path, map_location='cpu'))\n    else:\n        model.load_state_dict(torch.load(best_path, map_location='cpu'))\n\nfinal_loss, final_acc = evaluate(use_ema=True)\nprint(f\"Final (official CIFAR-10 test) — Loss: {final_loss:.4f}, Acc: {final_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T22:58:17.233014Z","iopub.execute_input":"2025-09-23T22:58:17.233782Z","iopub.status.idle":"2025-09-23T22:58:18.765274Z","shell.execute_reply.started":"2025-09-23T22:58:17.233753Z","shell.execute_reply":"2025-09-23T22:58:18.764481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 10: Prepare Kaggle test set (from test.7z) — Decompression (Kaggle Only)\n# =====================\n\n!apt -y install libarchive-dev\n!pip install -q libarchive\nimport libarchive.public\ncnt = 0\nfor entry in libarchive.public.file_pour('/kaggle/input/cifar-10/test.7z'):\n     cnt += 1\n     if cnt % 1000 == 0:\n         print(cnt)\nimport glob\nprint('Num files in test/:', len(glob.glob('test/*')))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T23:07:01.927467Z","iopub.execute_input":"2025-09-23T23:07:01.928013Z","iopub.status.idle":"2025-09-23T23:08:13.164453Z","shell.execute_reply.started":"2025-09-23T23:07:01.927985Z","shell.execute_reply":"2025-09-23T23:08:13.163468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Cell 11: Kaggle test dataset & submission writer\n# =====================\nclass KaggleCIFAR10Test(torch.utils.data.Dataset):\n    def __init__(self, root='test', transform=None):\n        self.root = Path(root)\n        self.files = sorted(self.root.glob('*.png'), key=lambda p: int(p.stem))\n        self.transform = transform if transform is not None else valid_transform\n\n    def __len__(self):\n        return len(self.files)\n\n    def __getitem__(self, idx):\n        p = self.files[idx]\n        img = Image.open(p).convert('RGB')\n        if self.transform:\n            img = self.transform(img)\n        return img, int(p.stem)\n\n@torch.no_grad()\ndef predict_test_and_write_submission(model_to_use=None, out_csv='submission.csv'):\n    net = model_to_use if model_to_use is not None else (ema_model if cfg.use_ema else model)\n    net.eval()\n\n    test_ds = KaggleCIFAR10Test(root='test', transform=valid_transform)\n    test_loader = DataLoader(test_ds, batch_size=cfg.batch_size, shuffle=False,\n                             num_workers=cfg.num_workers, pin_memory=True)\n\n    ids = []\n    labels = []\n    for images, idnums in test_loader:\n        images = images.to(DEVICE)\n        with torch.cuda.amp.autocast(enabled=(DEVICE.type=='cuda')):\n            logits = net(images)\n            preds = logits.argmax(1).detach().cpu().numpy()\n        ids.extend(idnums.numpy().tolist())\n        labels.extend([cfg.classes[p] for p in preds])\n\n    # Write submission\n    import pandas as pd\n    df = pd.DataFrame({'id': ids, 'label': labels})\n    df = df.sort_values('id')\n    df.to_csv(out_csv, index=False)\n    print('Wrote:', out_csv, 'with shape', df.shape)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T23:14:03.515417Z","iopub.execute_input":"2025-09-23T23:14:03.515733Z","iopub.status.idle":"2025-09-23T23:14:03.524450Z","shell.execute_reply.started":"2025-09-23T23:14:03.515709Z","shell.execute_reply":"2025-09-23T23:14:03.523642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predict_test_and_write_submission(out_csv=cfg.submission_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T23:14:24.051535Z","iopub.execute_input":"2025-09-23T23:14:24.052100Z","iopub.status.idle":"2025-09-23T23:15:35.464488Z","shell.execute_reply.started":"2025-09-23T23:14:24.052072Z","shell.execute_reply":"2025-09-23T23:15:35.463630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -lh submission.csv\n!head submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-23T23:17:44.707950Z","iopub.execute_input":"2025-09-23T23:17:44.708692Z","iopub.status.idle":"2025-09-23T23:17:44.995660Z","shell.execute_reply.started":"2025-09-23T23:17:44.708661Z","shell.execute_reply":"2025-09-23T23:17:44.994914Z"}},"outputs":[],"execution_count":null}]}