{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.system(\"pip install py7zr -q\")\n\nimport py7zr\n\nbase = \"/kaggle/input/competitions/cifar-10\"\nwork = \"/kaggle/working\"\n\nfor archive in [\"train.7z\"]:\n    path = os.path.join(base, archive)\n    print(f\"Extracting {archive} ...\")\n    with py7zr.SevenZipFile(path, mode='r') as z:\n        z.extractall(path=work)\n\nprint(\"Done.\")\nprint(os.listdir(work))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:10.579152Z","iopub.execute_input":"2026-09-22T20:13:10.580031Z","iopub.status.idle":"2026-09-22T20:13:24.689509Z","shell.execute_reply.started":"2026-09-22T20:13:10.579972Z","shell.execute_reply":"2026-09-22T20:13:24.68768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data explore","metadata":{}},{"cell_type":"code","source":"import glob\nprint(\"train:\", len(glob.glob(f\"{work}/train/*\")))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.690133Z","iopub.status.idle":"2026-09-22T20:13:24.690492Z","shell.execute_reply.started":"2026-09-22T20:13:24.690306Z","shell.execute_reply":"2026-09-22T20:13:24.690363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain_labels = pd.read_csv(f\"{base}/trainLabels.csv\")\nsample_sub = pd.read_csv(f\"{base}/sampleSubmission.csv\")\n\nprint(\"train_labels shape:\", train_labels.shape)\nprint(train_labels.head())\nprint()\nprint(\"sample_submission shape:\", sample_sub.shape)\nprint(sample_sub.head())\n\nprint(\"\\nTrain Class distribution:\")\nprint(train_labels['label'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.691781Z","iopub.status.idle":"2026-09-22T20:13:24.692152Z","shell.execute_reply.started":"2026-09-22T20:13:24.69197Z","shell.execute_reply":"2026-09-22T20:13:24.691992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom PIL import Image\n\ntrain_img_dir = os.path.join(work, \"train\")\nprint(\"Train images samples:\")\nfig, axes = plt.subplots(2, 5, figsize=(12, 5))\nfor i, ax in enumerate(axes.flat):\n    row = train_labels.iloc[i]\n    img_path = os.path.join(train_img_dir, f\"{row['id']}.png\")\n    img = Image.open(img_path)\n    ax.imshow(img)\n    ax.set_title(row['label'])\n    ax.axis('off')\nplt.tight_layout()\nplt.show()\n\n# Check image properties\nsample_img = Image.open(os.path.join(train_img_dir, f\"{train_labels.iloc[0]['id']}.png\"))\nprint(\"Image size:\", sample_img.size)\nprint(\"Image mode:\", sample_img.mode)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.693565Z","iopub.status.idle":"2026-09-22T20:13:24.693891Z","shell.execute_reply.started":"2026-09-22T20:13:24.693747Z","shell.execute_reply":"2026-09-22T20:13:24.693772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels['label'].value_counts().plot(kind='bar', figsize=(8,4))\nplt.title(\"Class distribution in training set\")\nplt.ylabel(\"count\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.695026Z","iopub.status.idle":"2026-09-22T20:13:24.695396Z","shell.execute_reply.started":"2026-09-22T20:13:24.695179Z","shell.execute_reply":"2026-09-22T20:13:24.695195Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data load","metadata":{}},{"cell_type":"markdown","source":"#### K fold split","metadata":{}},{"cell_type":"code","source":"import os, random, time, glob, shutil\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\n\n# --- Paths ---\nbase = \"/kaggle/input/competitions/cifar-10\"\nwork = \"/kaggle/working\"\nTRAIN_IMG_DIR = f\"{work}/train\"\nTEST_IMG_DIR  = f\"{work}/test\"\n\n# --- Hyperparameters learned from the earlier sweep (Adam) ---\nBEST_LR = 5e-4\nBEST_WD = 0.0\nBATCH_SIZE = 128          # same batch size the sweep used, so the learned LR still applies\n\n# --- Epoch search ---\nEPOCH_CANDIDATES = [20, 30, 40]\nK = 3\n\n# --- Other settings ---\nLABEL_SMOOTHING = 0.1\nNUM_WORKERS = 0           # images are preloaded into RAM, so workers aren't needed\nSEED = 42\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nUSE_AMP = device.type == \"cuda\"   # mixed precision: ~2-3x faster on GPU\ntorch.backends.cudnn.benchmark = True\nprint(\"Using device:\", device, \"| AMP:\", USE_AMP)\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nseed_everything(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.696833Z","iopub.status.idle":"2026-09-22T20:13:24.697206Z","shell.execute_reply.started":"2026-09-22T20:13:24.697017Z","shell.execute_reply":"2026-09-22T20:13:24.697042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels = pd.read_csv(f\"{base}/trainLabels.csv\")\n\nclasses = sorted(train_labels[\"label\"].unique())\nclass_to_idx = {c: i for i, c in enumerate(classes)}\nidx_to_class = {i: c for c, i in class_to_idx.items()}\ny_all = train_labels[\"label\"].map(class_to_idx).to_numpy(dtype=np.int64)\nprint(\"Classes:\", class_to_idx)\n\n# Positions (row numbers) into train_labels / X_all\nall_pos = np.arange(len(train_labels))\ntrain_pos, val_pos = train_test_split(\n    all_pos, test_size=0.1, stratify=y_all, random_state=SEED\n)\nprint(\"train:\", len(train_pos), \"| val (holdout):\", len(val_pos))\n\n# 3 folds on the train part only; val_pos is never used to choose anything\nskf = StratifiedKFold(n_splits=K, shuffle=True, random_state=SEED)\nfolds = [(train_pos[tr], train_pos[va]) for tr, va in skf.split(train_pos, y_all[train_pos])]\nprint(\"folds (train, val):\", [(len(a), len(b)) for a, b in folds])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_images(ids, img_dir, desc=\"loading images\"):\n    images = np.empty((len(ids), 32, 32, 3), dtype=np.uint8)\n    for i, img_id in enumerate(tqdm(ids, desc=desc)):\n        with Image.open(os.path.join(img_dir, f\"{img_id}.png\")) as img:\n            images[i] = np.asarray(img.convert(\"RGB\"))\n    return images\n\n# Row i of X_all is the image for train_labels row i\nX_all = load_images(train_labels[\"id\"].to_numpy(), TRAIN_IMG_DIR, desc=\"loading train images\")\nprint(\"X_all:\", X_all.shape, X_all.dtype, f\"{X_all.nbytes / 1e6:.0f} MB\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Dataset python class","metadata":{}},{"cell_type":"code","source":"CIFAR_MEAN = (0.4914, 0.4822, 0.4465)\nCIFAR_STD  = (0.2470, 0.2435, 0.2616)\n\ntrain_transform = T.Compose([\n    T.RandomCrop(32, padding=4, padding_mode=\"reflect\"),\n    T.RandomHorizontalFlip(),\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\neval_transform = T.Compose([\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\nclass ArrayDataset(Dataset):\n    \"\"\"Serves images from an in-memory uint8 array of shape (N, 32, 32, 3).\"\"\"\n    def __init__(self, images, labels=None, transform=None):\n        self.images = images\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, i):\n        img = Image.fromarray(self.images[i])\n        if self.transform is not None:\n            img = self.transform(img)\n        if self.labels is None:          # test set: no labels\n            return img\n        return img, int(self.labels[i])\n\ndef make_loader(positions, train):\n    ds = ArrayDataset(X_all[positions], y_all[positions],\n                      train_transform if train else eval_transform)\n    return DataLoader(ds, batch_size=BATCH_SIZE if train else 512, shuffle=train,\n                      num_workers=NUM_WORKERS, pin_memory=(device.type == \"cuda\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.69839Z","iopub.status.idle":"2026-09-22T20:13:24.698772Z","shell.execute_reply.started":"2026-09-22T20:13:24.698581Z","shell.execute_reply":"2026-09-22T20:13:24.698604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model architecture","metadata":{}},{"cell_type":"code","source":"def conv_bn(c_in, c_out, pool=False):\n    layers = [\n        nn.Conv2d(c_in, c_out, kernel_size=3, padding=1, bias=False),\n        nn.BatchNorm2d(c_out),\n        nn.ReLU(inplace=True),\n    ]\n    if pool:\n        layers.append(nn.MaxPool2d(2))\n    return nn.Sequential(*layers)\n\nclass Residual(nn.Module):\n    def __init__(self, channels):\n        super().__init__()\n        self.block = nn.Sequential(conv_bn(channels, channels), conv_bn(channels, channels))\n\n    def forward(self, x):\n        return x + self.block(x)       # skip connection\n\nclass CNN(nn.Module):\n    \"\"\"ResNet-9 style network for 32x32 images.\"\"\"\n    def __init__(self, num_classes=10):\n        super().__init__()\n        self.net = nn.Sequential(\n            conv_bn(3, 64),                  # 32x32\n            conv_bn(64, 128, pool=True),     # -> 16x16\n            Residual(128),\n            conv_bn(128, 256, pool=True),    # -> 8x8\n            conv_bn(256, 512, pool=True),    # -> 4x4\n            Residual(512),\n            nn.AdaptiveMaxPool2d(1),         # -> 1x1\n            nn.Flatten(),\n            nn.Dropout(0.2),\n            nn.Linear(512, num_classes),\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\nn_params = sum(p.numel() for p in CNN().parameters())\nprint(f\"CNN parameters: {n_params / 1e6:.2f}M\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.701609Z","iopub.status.idle":"2026-09-22T20:13:24.701833Z","shell.execute_reply.started":"2026-09-22T20:13:24.701724Z","shell.execute_reply":"2026-09-22T20:13:24.701737Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Train/eval loops","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, scheduler, scaler, criterion, desc=\"train\"):\n    model.train()\n    total_loss, correct, total = 0.0, 0, 0\n    pbar = tqdm(loader, desc=desc, leave=False)\n    for images, labels in pbar:\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n\n        optimizer.zero_grad(set_to_none=True)\n        with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=USE_AMP):\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()                 # OneCycleLR steps every batch\n\n        total_loss += loss.item() * images.size(0)\n        correct += (outputs.argmax(1) == labels).sum().item()\n        total += images.size(0)\n        pbar.set_postfix(loss=f\"{total_loss / total:.3f}\", acc=f\"{correct / total:.3f}\")\n    return total_loss / total, correct / total\n\n@torch.no_grad()\ndef evaluate(model, loader, criterion, desc=\"eval\"):\n    model.eval()\n    total_loss, correct, total = 0.0, 0, 0\n    for images, labels in tqdm(loader, desc=desc, leave=False):\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=USE_AMP):\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n        total_loss += loss.item() * images.size(0)\n        correct += (outputs.argmax(1) == labels).sum().item()\n        total += images.size(0)\n    return total_loss / total, correct / total\n\ndef run_training(train_positions, epochs, eval_sets=None, tag=\"\"):\n    \"\"\"Train a fresh CNN for `epochs` with a one-cycle LR schedule.\n    eval_sets: optional dict {name: positions}, evaluated after every epoch.\"\"\"\n    seed_everything(SEED)\n    train_loader = make_loader(train_positions, train=True)\n    eval_loaders = {name: make_loader(pos, train=False) for name, pos in (eval_sets or {}).items()}\n\n    model = CNN().to(device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=BEST_LR, weight_decay=BEST_WD)\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n        optimizer, max_lr=BEST_LR, epochs=epochs, steps_per_epoch=len(train_loader)\n    )\n    scaler = torch.amp.GradScaler(\"cuda\", enabled=USE_AMP)\n    criterion = nn.CrossEntropyLoss(label_smoothing=LABEL_SMOOTHING)\n\n    history = []\n    for epoch in range(epochs):\n        t0 = time.time()\n        tr_loss, tr_acc = train_one_epoch(model, train_loader, optimizer, scheduler, scaler, criterion,\n                                          desc=f\"{tag} epoch {epoch + 1}/{epochs}\")\n        row = {\"epoch\": epoch + 1, \"train_loss\": tr_loss, \"train_acc\": tr_acc}\n        for name, loader in eval_loaders.items():\n            _, row[f\"{name}_acc\"] = evaluate(model, loader, criterion, desc=f\"{tag} eval {name}\")\n        history.append(row)\n\n        msg = f\"{tag} epoch {epoch + 1:>2}/{epochs}: train_acc={tr_acc:.4f}\"\n        for name in eval_loaders:\n            msg += f\"  {name}_acc={row[f'{name}_acc']:.4f}\"\n        msg += f\"  ({time.time() - t0:.0f}s)\"\n        print(msg)\n    return model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.702564Z","iopub.status.idle":"2026-09-22T20:13:24.702838Z","shell.execute_reply.started":"2026-09-22T20:13:24.702714Z","shell.execute_reply":"2026-09-22T20:13:24.702728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## training","metadata":{}},{"cell_type":"code","source":"cv_rows = []\nall_histories = {}\n\nfor E in EPOCH_CANDIDATES:\n    for f, (tr_p, va_p) in enumerate(folds):\n        print(f\"\\n=== {E} epochs | fold {f + 1}/{K} ===\")\n        model, hist = run_training(tr_p, E, eval_sets={\"fold_val\": va_p, \"holdout\": val_pos},\n                                   tag=f\"[E{E} F{f + 1}]\")\n        all_histories[(E, f)] = hist\n        cv_rows.append({\"epochs\": E, \"fold\": f + 1,\n                        \"fold_val_acc\": hist[-1][\"fold_val_acc\"],\n                        \"holdout_acc\": hist[-1][\"holdout_acc\"]})\n        del model\n        if device.type == \"cuda\":\n            torch.cuda.empty_cache()\n\ncv_df = pd.DataFrame(cv_rows)\ncv_summary = cv_df.groupby(\"epochs\")[[\"fold_val_acc\", \"holdout_acc\"]].agg([\"mean\", \"std\"])\nprint(\"\\n=== CV summary ===\")\nprint(cv_summary.round(4))\n\n# Best epoch count = highest mean fold-validation accuracy (holdout is only reported)\nBEST_EPOCHS = int(cv_df.groupby(\"epochs\")[\"fold_val_acc\"].mean().idxmax())\nprint(f\"\\nBest number of epochs: {BEST_EPOCHS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.70715Z","iopub.status.idle":"2026-09-22T20:13:24.707564Z","shell.execute_reply.started":"2026-09-22T20:13:24.70736Z","shell.execute_reply":"2026-09-22T20:13:24.707384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Optional manual override (e.g. if you skip the CV cell) ---\nUSE_MANUAL_EPOCHS = False\nMANUAL_EPOCHS = 40\n\nif USE_MANUAL_EPOCHS:\n    BEST_EPOCHS = MANUAL_EPOCHS\nprint(f\"Final training will use {BEST_EPOCHS} epochs \"\n      f\"(lr={BEST_LR}, wd={BEST_WD}, batch={BATCH_SIZE})\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.70889Z","iopub.status.idle":"2026-09-22T20:13:24.709257Z","shell.execute_reply.started":"2026-09-22T20:13:24.709068Z","shell.execute_reply":"2026-09-22T20:13:24.709091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(9, 4))\nfor E in EPOCH_CANDIDATES:\n    mean_curve = np.mean([[h[\"fold_val_acc\"] for h in all_histories[(E, f)]] for f in range(K)], axis=0)\n    plt.plot(range(1, E + 1), mean_curve, label=f\"{E}-epoch schedule\")\nplt.xlabel(\"epoch\"); plt.ylabel(\"mean fold val accuracy\")\nplt.title(\"3-fold CV: validation accuracy per schedule length\")\nplt.legend(); plt.grid(alpha=0.3)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.710981Z","iopub.status.idle":"2026-09-22T20:13:24.711551Z","shell.execute_reply.started":"2026-09-22T20:13:24.711342Z","shell.execute_reply":"2026-09-22T20:13:24.711368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Final training","metadata":{}},{"cell_type":"code","source":"full_pos = np.concatenate([train_pos, val_pos])     # train + val = all 50,000 images\nprint(\"Final training on\", len(full_pos), \"images for\", BEST_EPOCHS, \"epochs\")\n\nfinal_model, final_history = run_training(full_pos, BEST_EPOCHS, eval_sets=None, tag=\"[final]\")\n\ntorch.save(final_model.state_dict(), f\"{work}/final_model.pt\")\nprint(\"Saved weights to\", f\"{work}/final_model.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T20:13:24.712483Z","iopub.status.idle":"2026-09-22T20:13:24.712794Z","shell.execute_reply.started":"2026-09-22T20:13:24.712674Z","shell.execute_reply":"2026-09-22T20:13:24.712691Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"# --- Extract test.7z (skipped if already extracted) ---\nif len(glob.glob(f\"{TEST_IMG_DIR}/*.png\")) != 300000:\n    import py7zr\n    print(\"Extracting test.7z ...\")\n    with py7zr.SevenZipFile(f\"{base}/test.7z\", mode=\"r\") as z:\n        z.extractall(path=work)\nprint(\"test images:\", len(glob.glob(f\"{TEST_IMG_DIR}/*.png\")))\n\n# --- Load test images in the exact order of sampleSubmission.csv ---\nsample_sub = pd.read_csv(f\"{base}/sampleSubmission.csv\")\ntest_ids = sample_sub[\"id\"].to_numpy()\nX_test = load_images(test_ids, TEST_IMG_DIR, desc=\"loading test images\")\n\ntest_loader = DataLoader(ArrayDataset(X_test, labels=None, transform=eval_transform),\n                         batch_size=512, shuffle=False, num_workers=NUM_WORKERS,\n                         pin_memory=(device.type == \"cuda\"))\n\n# --- Predict with test-time augmentation (original + horizontal flip) ---\n@torch.no_grad()\ndef predict(model, loader, tta=True):\n    model.eval()\n    preds = []\n    for images in tqdm(loader, desc=\"inference\"):\n        images = images.to(device, non_blocking=True)\n        with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=USE_AMP):\n            probs = model(images).float().softmax(dim=1)\n            if tta:\n                probs += model(torch.flip(images, dims=[3])).float().softmax(dim=1)\n        preds.append(probs.argmax(dim=1).cpu())\n    return torch.cat(preds).numpy()\n\ntest_preds = predict(final_model, test_loader, tta=True)\n\n# --- Build and save submission.csv ---\nsubmission = pd.DataFrame({\"id\": test_ids, \"label\": np.array(classes)[test_preds]})\nassert submission.shape == sample_sub.shape\nassert (submission[\"id\"].to_numpy() == sample_sub[\"id\"].to_numpy()).all()\nsubmission.to_csv(f\"{work}/submission.csv\", index=False)\n\nprint(submission.head())\nprint(\"\\nPredicted class distribution:\")\nprint(submission[\"label\"].value_counts())\nprint(\"\\nSaved to\", f\"{work}/submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}