{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.system(\"pip install py7zr -q\")\n\nimport py7zr\n\nbase = \"/kaggle/input/competitions/cifar-10\"\nwork = \"/kaggle/working\"\n\nfor archive in [\"train.7z\"]:\n    path = os.path.join(base, archive)\n    print(f\"Extracting {archive} ...\")\n    with py7zr.SevenZipFile(path, mode='r') as z:\n        z.extractall(path=work)\n\nprint(\"Done.\")\nprint(os.listdir(work))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:43:49.910943Z","iopub.execute_input":"2026-10-06T17:43:49.91181Z","iopub.status.idle":"2026-10-06T17:44:31.850017Z","shell.execute_reply.started":"2026-10-06T17:43:49.911779Z","shell.execute_reply":"2026-10-06T17:44:31.849253Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Explore","metadata":{}},{"cell_type":"code","source":"import glob\nprint(\"train:\", len(glob.glob(f\"{work}/train/*\")))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:31.8516Z","iopub.execute_input":"2026-10-06T17:44:31.851959Z","iopub.status.idle":"2026-10-06T17:44:31.95377Z","shell.execute_reply.started":"2026-10-06T17:44:31.851935Z","shell.execute_reply":"2026-10-06T17:44:31.952896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain_labels = pd.read_csv(f\"{base}/trainLabels.csv\")\nsample_sub = pd.read_csv(f\"{base}/sampleSubmission.csv\")\n\nprint(\"train_labels shape:\", train_labels.shape)\nprint(train_labels.head())\nprint()\nprint(\"sample_submission shape:\", sample_sub.shape)\nprint(sample_sub.head())\n\nprint(\"\\nTrain Class distribution:\")\nprint(train_labels['label'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:31.954853Z","iopub.execute_input":"2026-10-06T17:44:31.955277Z","iopub.status.idle":"2026-10-06T17:44:32.555104Z","shell.execute_reply.started":"2026-10-06T17:44:31.955249Z","shell.execute_reply":"2026-10-06T17:44:32.554114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom PIL import Image\n\ntrain_img_dir = os.path.join(work, \"train\")\nprint(\"Train images samples:\")\nfig, axes = plt.subplots(2, 5, figsize=(12, 5))\nfor i, ax in enumerate(axes.flat):\n    row = train_labels.iloc[i]\n    img_path = os.path.join(train_img_dir, f\"{row['id']}.png\")\n    img = Image.open(img_path)\n    ax.imshow(img)\n    ax.set_title(row['label'])\n    ax.axis('off')\nplt.tight_layout()\nplt.show()\n\n# Check image properties\nsample_img = Image.open(os.path.join(train_img_dir, f\"{train_labels.iloc[0]['id']}.png\"))\nprint(\"Image size:\", sample_img.size)\nprint(\"Image mode:\", sample_img.mode)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:32.557468Z","iopub.execute_input":"2026-10-06T17:44:32.558064Z","iopub.status.idle":"2026-10-06T17:44:33.180481Z","shell.execute_reply.started":"2026-10-06T17:44:32.558034Z","shell.execute_reply":"2026-10-06T17:44:33.179665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels['label'].value_counts().plot(kind='bar', figsize=(8,4))\nplt.title(\"Class distribution in training set\")\nplt.ylabel(\"count\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:33.181885Z","iopub.execute_input":"2026-10-06T17:44:33.182253Z","iopub.status.idle":"2026-10-06T17:44:33.418795Z","shell.execute_reply.started":"2026-10-06T17:44:33.182212Z","shell.execute_reply":"2026-10-06T17:44:33.418008Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Load","metadata":{}},{"cell_type":"markdown","source":"### Train / holdout split","metadata":{}},{"cell_type":"code","source":"import os, random, time, glob, math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\n\n# --- Paths ---\nbase = \"/kaggle/input/competitions/cifar-10\"\nwork = \"/kaggle/working\"\nTRAIN_IMG_DIR = f\"{work}/train\"\nTEST_IMG_DIR  = f\"{work}/test\"\n\n# --- Optimizer: AdamW (decoupled weight decay) ---\nLR = 1e-3                 # peak LR, reached at the end of warmup\nWD = 5e-2                 # decoupled weight decay (not applied to BN weights / biases)\nBATCH_SIZE = 128\n\n# --- LR schedule: linear warmup -> cosine annealing ---\nEPOCHS = 150\nWARMUP_EPOCHS = 5\nMIN_LR_RATIO = 0.01       # cosine decays from LR down to LR * MIN_LR_RATIO\n\n# --- Advanced augmentation: CutMix (on top of crop + flip) ---\nCUTMIX_ALPHA = 1.0        # mixing ratio lambda ~ Beta(alpha, alpha)\nCUTMIX_PROB = 0.5         # fraction of batches that get CutMix\nCUTMIX_OFF_EPOCHS = 10    # train the last N epochs on clean images\n\n# --- Run control ---\nRUN_HOLDOUT_CHECK = True  # train once on 45k and score the 5k holdout before the final run\n\n# --- Other settings ---\nLABEL_SMOOTHING = 0.1\nNUM_WORKERS = min(4, os.cpu_count() or 1)   # parallel CPU workers for crop/flip\nSEED = 42\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nUSE_AMP = device.type == \"cuda\"   # mixed precision: ~2-3x faster on GPU\ntorch.backends.cudnn.benchmark = True\nprint(\"Using device:\", device, \"| AMP:\", USE_AMP, \"| workers:\", NUM_WORKERS)\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nseed_everything(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:33.419882Z","iopub.execute_input":"2026-10-06T17:44:33.420271Z","iopub.status.idle":"2026-10-06T17:44:47.156528Z","shell.execute_reply.started":"2026-10-06T17:44:33.420234Z","shell.execute_reply":"2026-10-06T17:44:47.1557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels = pd.read_csv(f\"{base}/trainLabels.csv\")\n\nclasses = sorted(train_labels[\"label\"].unique())\nclass_to_idx = {c: i for i, c in enumerate(classes)}\nidx_to_class = {i: c for c, i in class_to_idx.items()}\ny_all = train_labels[\"label\"].map(class_to_idx).to_numpy(dtype=np.int64)\nprint(\"Classes:\", class_to_idx)\n\n# Positions (row numbers) into train_labels / X_all\nall_pos = np.arange(len(train_labels))\ntrain_pos, val_pos = train_test_split(\n    all_pos, test_size=0.1, stratify=y_all, random_state=SEED\n)\nprint(\"train:\", len(train_pos), \"| val (holdout):\", len(val_pos))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:47.157759Z","iopub.execute_input":"2026-10-06T17:44:47.158322Z","iopub.status.idle":"2026-10-06T17:44:47.233734Z","shell.execute_reply.started":"2026-10-06T17:44:47.158294Z","shell.execute_reply":"2026-10-06T17:44:47.23272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_images(ids, img_dir, desc=\"loading images\"):\n    images = np.empty((len(ids), 32, 32, 3), dtype=np.uint8)\n    for i, img_id in enumerate(tqdm(ids, desc=desc)):\n        with Image.open(os.path.join(img_dir, f\"{img_id}.png\")) as img:\n            images[i] = np.asarray(img.convert(\"RGB\"))\n    return images\n\n# Row i of X_all is the image for train_labels row i\nX_all = load_images(train_labels[\"id\"].to_numpy(), TRAIN_IMG_DIR, desc=\"loading train images\")\nprint(\"X_all:\", X_all.shape, X_all.dtype, f\"{X_all.nbytes / 1e6:.0f} MB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:47.234941Z","iopub.execute_input":"2026-10-06T17:44:47.235427Z","iopub.status.idle":"2026-10-06T17:44:57.655029Z","shell.execute_reply.started":"2026-10-06T17:44:47.235393Z","shell.execute_reply":"2026-10-06T17:44:57.653853Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Dataset class","metadata":{}},{"cell_type":"code","source":"CIFAR_MEAN = (0.4914, 0.4822, 0.4465)\nCIFAR_STD  = (0.2470, 0.2435, 0.2616)\n\n# Per-image augmentation on CPU. CutMix is applied per batch on the GPU in the training loop.\ntrain_transform = T.Compose([\n    T.RandomCrop(32, padding=4, padding_mode=\"reflect\"),\n    T.RandomHorizontalFlip(),\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\neval_transform = T.Compose([\n    T.ToTensor(),\n    T.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\nclass ArrayDataset(Dataset):\n    \"\"\"Serves images from an in-memory uint8 array of shape (N, 32, 32, 3).\"\"\"\n    def __init__(self, images, labels=None, transform=None):\n        self.images = images\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, i):\n        img = Image.fromarray(self.images[i])\n        if self.transform is not None:\n            img = self.transform(img)\n        if self.labels is None:          # test set: no labels\n            return img\n        return img, int(self.labels[i])\n\ndef make_loader(positions, train):\n    ds = ArrayDataset(X_all[positions], y_all[positions],\n                      train_transform if train else eval_transform)\n    return DataLoader(ds, batch_size=BATCH_SIZE if train else 512, shuffle=train,\n                      drop_last=train,                       # equal-size batches for CutMix\n                      num_workers=NUM_WORKERS,\n                      persistent_workers=NUM_WORKERS > 0,\n                      pin_memory=(device.type == \"cuda\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:57.656406Z","iopub.execute_input":"2026-10-06T17:44:57.656782Z","iopub.status.idle":"2026-10-06T17:44:57.666986Z","shell.execute_reply.started":"2026-10-06T17:44:57.656757Z","shell.execute_reply":"2026-10-06T17:44:57.666022Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Architecture","metadata":{}},{"cell_type":"code","source":"class ConvBNAct(nn.Sequential):\n    \"\"\"k x k conv -> BatchNorm -> GELU (used only in the stem).\"\"\"\n    def __init__(self, c_in, c_out, k=3):\n        super().__init__(\n            nn.Conv2d(c_in, c_out, k, padding=k // 2, bias=False),\n            nn.BatchNorm2d(c_out),\n            nn.GELU(),\n        )\n\nclass SqueezeExcite(nn.Module):\n    \"\"\"Channel attention: global context decides how much each channel is kept.\"\"\"\n    def __init__(self, dim, ratio=0.25):\n        super().__init__()\n        hidden = max(8, int(dim * ratio))\n        self.gate = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Conv2d(dim, hidden, 1),\n            nn.GELU(),\n            nn.Conv2d(hidden, dim, 1),\n            nn.Sigmoid(),\n        )\n\n    def forward(self, x):\n        return x * self.gate(x)\n\nclass MultiScaleDW(nn.Module):\n    \"\"\"Depthwise spatial mixing at several scales: channels are split into groups,\n    each group gets its own kernel size (default: half 3x3, half 7x7).\"\"\"\n    def __init__(self, dim, kernels=(3, 7)):\n        super().__init__()\n        n = len(kernels)\n        self.splits = [dim // n] * (n - 1) + [dim - (dim // n) * (n - 1)]\n        self.convs = nn.ModuleList(\n            nn.Conv2d(c, c, k, padding=k // 2, groups=c, bias=False)\n            for c, k in zip(self.splits, kernels)\n        )\n\n    def forward(self, x):\n        chunks = torch.split(x, self.splits, dim=1)\n        return torch.cat([conv(c) for conv, c in zip(self.convs, chunks)], dim=1)\n\nclass MixBlock(nn.Module):\n    \"\"\"Two decoupled mixing steps:\n       1) spatial mixing  - multi-scale depthwise conv + SE attention (no channel mixing)\n       2) channel mixing  - pointwise MLP (1x1 -> expand -> 1x1), no spatial mixing\"\"\"\n    def __init__(self, dim, expansion=2, kernels=(3, 7)):\n        super().__init__()\n        self.spatial = nn.Sequential(\n            MultiScaleDW(dim, kernels),\n            nn.GELU(),\n            nn.BatchNorm2d(dim),\n            SqueezeExcite(dim),\n        )\n        hidden = dim * expansion\n        self.channel = nn.Sequential(\n            nn.Conv2d(dim, hidden, 1, bias=False),\n            nn.BatchNorm2d(hidden),\n            nn.GELU(),\n            nn.Conv2d(hidden, dim, 1, bias=False),\n            nn.BatchNorm2d(dim),\n        )\n        nn.init.zeros_(self.channel[-1].weight)   # channel-MLP branch starts as a no-op\n\n    def forward(self, x):\n        x = x + self.spatial(x)\n        x = x + self.channel(x)\n        return x\n\nclass Downsample(nn.Sequential):\n    \"\"\"2x2 stride-2 'patch merging' conv: halves the resolution and widens the channels.\"\"\"\n    def __init__(self, c_in, c_out):\n        super().__init__(\n            nn.Conv2d(c_in, c_out, kernel_size=2, stride=2, bias=False),\n            nn.BatchNorm2d(c_out),\n        )\n\nclass CNN(nn.Module):\n    \"\"\"Multi-Scale Mixer network for 32x32 images.\n    stem (32x32) -> stage 1 (32x32) -> merge -> stage 2 (16x16) -> merge -> stage 3 (8x8) -> head\"\"\"\n    def __init__(self, num_classes=10, widths=(96, 192, 384), depths=(2, 3, 3),\n                 expansion=2, kernels=(3, 7), drop=0.1):\n        super().__init__()\n        self.stem = nn.Sequential(ConvBNAct(3, 32), ConvBNAct(32, widths[0]))\n        layers = []\n        for i, (w, d) in enumerate(zip(widths, depths)):\n            if i > 0:\n                layers.append(Downsample(widths[i - 1], w))\n            layers += [MixBlock(w, expansion, kernels) for _ in range(d)]\n        self.stages = nn.Sequential(*layers)\n        self.head = nn.Sequential(\n            nn.BatchNorm2d(widths[-1]),\n            nn.AdaptiveAvgPool2d(1),\n            nn.Flatten(),\n            nn.Dropout(drop),\n            nn.Linear(widths[-1], num_classes),\n        )\n\n    def forward(self, x):\n        return self.head(self.stages(self.stem(x)))\n\nn_params = sum(p.numel() for p in CNN().parameters())\nprint(f\"CNN parameters: {n_params / 1e6:.2f}M\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:57.669613Z","iopub.execute_input":"2026-10-06T17:44:57.669989Z","iopub.status.idle":"2026-10-06T17:44:57.793358Z","shell.execute_reply.started":"2026-10-06T17:44:57.669965Z","shell.execute_reply":"2026-10-06T17:44:57.792539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import inspect\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import FancyBboxPatch\n\n# Read the architecture settings straight from CNN's defaults, so the drawing stays in sync\n_sig = inspect.signature(CNN.__init__).parameters\nWIDTHS    = _sig[\"widths\"].default\nDEPTHS    = _sig[\"depths\"].default\nEXPANSION = _sig[\"expansion\"].default\nKERNELS   = _sig[\"kernels\"].default\n\nCOLORS = {\"io\": \"#e0e0e0\", \"stem\": \"#ffd8a8\", \"mix\": \"#a5d8ff\", \"down\": \"#ffc9c9\",\n          \"head\": \"#b2f2bb\", \"dw\": \"#d0bfff\", \"op\": \"#fff3bf\"}\n\ndef box(ax, x, y, w, h, text, color, fs=9):\n    \"\"\"Rounded rectangle centred at (x, y) with a text label.\"\"\"\n    ax.add_patch(FancyBboxPatch((x - w / 2, y - h / 2), w, h, boxstyle=\"round,pad=0.02,rounding_size=0.08\",\n                                fc=color, ec=\"black\", lw=1.2))\n    ax.text(x, y, text, ha=\"center\", va=\"center\", fontsize=fs)\n\ndef arrow(ax, x0, y0, x1, y1, text=None):\n    ax.annotate(\"\", xy=(x1, y1), xytext=(x0, y0),\n                arrowprops=dict(arrowstyle=\"-|>\", lw=1.2, color=\"black\"))\n    if text:\n        ax.text(x0 + 0.12, (y0 + y1) / 2, text, ha=\"left\", va=\"center\", fontsize=8, color=\"dimgray\")\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 10))\n\n# ---------------- Left: whole network ----------------\nitems = [(\"Input image\", \"io\", \"3×32×32\")]\nitems.append((f\"Stem\\nConv3×3-BN-GELU (3→32)\\nConv3×3-BN-GELU (32→{WIDTHS[0]})\", \"stem\", f\"{WIDTHS[0]}×32×32\"))\nres = 32\nfor i, (w, d) in enumerate(zip(WIDTHS, DEPTHS)):\n    if i > 0:\n        res //= 2\n        items.append((f\"Downsample\\nConv2×2 stride 2 + BN ({WIDTHS[i-1]}→{w})\", \"down\", f\"{w}×{res}×{res}\"))\n    items.append((f\"Stage {i + 1}\\nMixBlock × {d}   (C = {w})\", \"mix\", f\"{w}×{res}×{res}\"))\nitems.append((\"Head\\nBN → Global AvgPool → Dropout → Linear\", \"head\", \"10 logits\"))\n\nn = len(items)\nys = [n - 1 - k for k in range(n)]\nfor (text, kind, shape), y in zip(items, ys):\n    box(ax1, 0, y, 3.4, 0.72, text, COLORS[kind])\nfor k in range(n - 1):\n    arrow(ax1, 0, ys[k] - 0.36, 0, ys[k + 1] + 0.36, items[k][2])\nax1.text(0.12, ys[-1] - 0.55, items[-1][2], ha=\"left\", va=\"center\", fontsize=8, color=\"dimgray\")\nn_params = sum(p.numel() for p in CNN().parameters())\nax1.set_title(f\"Multi-Scale Mixer network ({n_params / 1e6:.2f}M parameters)\", fontsize=12, fontweight=\"bold\")\nax1.set_xlim(-2.4, 2.4); ax1.set_ylim(-1, n); ax1.axis(\"off\")\n\n# ---------------- Right: one MixBlock ----------------\nhid = f\"{EXPANSION}C\"\nx = 0\nrows = {\"in\": 11, \"dw\": 10, \"cat\": 9, \"gelu1\": 8.1, \"bn1\": 7.3, \"se\": 6.5, \"add1\": 5.5,\n        \"pw1\": 4.5, \"pw2\": 3.7, \"pw3\": 2.9, \"pw4\": 2.1, \"add2\": 1.1, \"out\": 0.2}\n\nbox(ax2, x, rows[\"in\"], 2.2, 0.55, \"input  x   (C×H×W)\", COLORS[\"io\"])\nnk = len(KERNELS)\nxs = [0.0] if nk == 1 else [(-1 + 2 * j / (nk - 1)) * 0.9 for j in range(nk)]\nfor xk, k in zip(xs, KERNELS):\n    box(ax2, xk, rows[\"dw\"], 1.3, 0.6, f\"DW {k}×{k}\\n(C/{nk} ch)\", COLORS[\"dw\"])\n    arrow(ax2, x, rows[\"in\"] - 0.28, xk, rows[\"dw\"] + 0.3)\n    arrow(ax2, xk, rows[\"dw\"] - 0.3, x, rows[\"cat\"] + 0.25)\nax2.text(x, rows[\"in\"] - 0.5, \"split channels\", ha=\"center\", fontsize=8, color=\"dimgray\")\nbox(ax2, x, rows[\"cat\"], 1.6, 0.5, \"Concat\", COLORS[\"op\"])\nbox(ax2, x, rows[\"gelu1\"], 1.6, 0.5, \"GELU\", COLORS[\"op\"])\nbox(ax2, x, rows[\"bn1\"], 1.6, 0.5, \"BatchNorm\", COLORS[\"op\"])\nbox(ax2, x, rows[\"se\"], 2.4, 0.55, \"Squeeze-Excite\\n(GAP → 1×1 → GELU → 1×1 → σ)\", COLORS[\"mix\"], fs=8)\nfor a, b in [(\"cat\", \"gelu1\"), (\"gelu1\", \"bn1\"), (\"bn1\", \"se\")]:\n    arrow(ax2, x, rows[a] - 0.25, x, rows[b] + 0.27)\n\nbox(ax2, x, rows[\"pw1\"], 2.2, 0.5, f\"Conv 1×1  (C → {hid})\", COLORS[\"stem\"])\nbox(ax2, x, rows[\"pw2\"], 2.2, 0.5, \"BatchNorm → GELU\", COLORS[\"op\"])\nbox(ax2, x, rows[\"pw3\"], 2.2, 0.5, f\"Conv 1×1  ({hid} → C)\", COLORS[\"stem\"])\nbox(ax2, x, rows[\"pw4\"], 2.2, 0.5, \"BatchNorm  (γ init = 0)\", COLORS[\"op\"])\nfor a, b in [(\"pw1\", \"pw2\"), (\"pw2\", \"pw3\"), (\"pw3\", \"pw4\")]:\n    arrow(ax2, x, rows[a] - 0.25, x, rows[b] + 0.25)\n\nfor key in (\"add1\", \"add2\"):  # the two \"+\" nodes\n    ax2.add_patch(plt.Circle((x, rows[key]), 0.22, fc=\"white\", ec=\"black\", lw=1.2))\n    ax2.text(x, rows[key], \"+\", ha=\"center\", va=\"center\", fontsize=14)\narrow(ax2, x, rows[\"se\"] - 0.28, x, rows[\"add1\"] + 0.22)\narrow(ax2, x, rows[\"add1\"] - 0.22, x, rows[\"pw1\"] + 0.25)\narrow(ax2, x, rows[\"pw4\"] - 0.25, x, rows[\"add2\"] + 0.22)\nbox(ax2, x, rows[\"out\"], 1.6, 0.45, \"output (C×H×W)\", COLORS[\"io\"])\narrow(ax2, x, rows[\"add2\"] - 0.22, x, rows[\"out\"] + 0.23)\n\n# skip connections (dashed, on the left)\nsx = -1.9\nfor top, bottom, start in [(rows[\"in\"], rows[\"add1\"], x - 1.1), (rows[\"add1\"], rows[\"add2\"], x - 0.22)]:\n    ax2.plot([start, sx, sx], [top, top, bottom], color=\"gray\", lw=1.2, ls=\"--\")\n    arrow(ax2, sx, bottom, x - 0.22, bottom)\n    ax2.text(sx - 0.15, (top + bottom) / 2, \"skip\", rotation=90, ha=\"center\", va=\"center\", fontsize=8, color=\"gray\")\n\nax2.text(2.0, (rows[\"dw\"] + rows[\"se\"]) / 2, \"spatial\\nmixing\", ha=\"center\", va=\"center\",\n         fontsize=10, fontweight=\"bold\", color=\"#5f3dc4\")\nax2.text(2.0, (rows[\"pw1\"] + rows[\"pw4\"]) / 2, \"channel\\nmixing\", ha=\"center\", va=\"center\",\n         fontsize=10, fontweight=\"bold\", color=\"#d9480f\")\nax2.set_title(\"MixBlock (one block)\", fontsize=12, fontweight=\"bold\")\nax2.set_xlim(-2.6, 2.6); ax2.set_ylim(-0.4, 11.6); ax2.axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:57.79445Z","iopub.execute_input":"2026-10-06T17:44:57.795215Z","iopub.status.idle":"2026-10-06T17:44:58.44999Z","shell.execute_reply.started":"2026-10-06T17:44:57.795145Z","shell.execute_reply":"2026-10-06T17:44:58.448947Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train/eval loop","metadata":{}},{"cell_type":"code","source":"# ---------- CutMix ----------\ndef cutmix(images, labels, alpha):\n    \"\"\"CutMix (Yun et al., 2019): paste a random box from a shuffled copy of the batch.\n    Returns mixed images, both label sets, and the area-based mixing ratio lam.\"\"\"\n    lam = np.random.beta(alpha, alpha)\n    perm = torch.randperm(images.size(0), device=images.device)\n    H, W = images.shape[2], images.shape[3]\n    cut = math.sqrt(1.0 - lam)\n    ch, cw = int(H * cut), int(W * cut)\n    cy, cx = np.random.randint(H), np.random.randint(W)\n    y1, y2 = max(cy - ch // 2, 0), min(cy + ch // 2, H)\n    x1, x2 = max(cx - cw // 2, 0), min(cx + cw // 2, W)\n    images[:, :, y1:y2, x1:x2] = images[perm, :, y1:y2, x1:x2]\n    lam = 1.0 - (y2 - y1) * (x2 - x1) / (H * W)   # exact ratio after clipping at the border\n    return images, labels, labels[perm], lam\n\n# ---------- Warmup + cosine annealing ----------\ndef lr_factor(step, total_steps, warmup_steps, min_ratio):\n    \"\"\"Multiplier on LR: linear 0 -> 1 during warmup, then cosine 1 -> min_ratio.\"\"\"\n    if step < warmup_steps:\n        return (step + 1) / warmup_steps\n    progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)\n    return min_ratio + (1.0 - min_ratio) * 0.5 * (1.0 + math.cos(math.pi * progress))\n\ndef make_scheduler(optimizer, total_steps, warmup_steps):\n    return torch.optim.lr_scheduler.LambdaLR(\n        optimizer, lambda s: lr_factor(s, total_steps, warmup_steps, MIN_LR_RATIO))\n\n# ---------- AdamW (no weight decay on BN weights and biases) ----------\ndef make_optimizer(model):\n    decay, no_decay = [], []\n    for p in model.parameters():\n        (no_decay if p.ndim <= 1 else decay).append(p)\n    return torch.optim.AdamW([{\"params\": decay, \"weight_decay\": WD},\n                              {\"params\": no_decay, \"weight_decay\": 0.0}], lr=LR)\n\n# ---------- Train / eval ----------\ndef train_one_epoch(model, loader, optimizer, scheduler, scaler, criterion, use_cutmix, desc=\"train\"):\n    model.train()\n    total_loss, correct, total = 0.0, 0.0, 0\n    pbar = tqdm(loader, desc=desc, leave=False)\n    for images, labels in pbar:\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n\n        mixed = use_cutmix and np.random.rand() < CUTMIX_PROB\n        if mixed:\n            images, y_a, y_b, lam = cutmix(images, labels, CUTMIX_ALPHA)\n\n        optimizer.zero_grad(set_to_none=True)\n        with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=USE_AMP):\n            outputs = model(images)\n            if mixed:\n                loss = lam * criterion(outputs, y_a) + (1 - lam) * criterion(outputs, y_b)\n            else:\n                loss = criterion(outputs, labels)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()                 # warmup + cosine steps every batch\n\n        preds = outputs.argmax(1)\n        if mixed:   # accuracy weighted by how much of each label is in the image\n            correct += (lam * (preds == y_a).float() + (1 - lam) * (preds == y_b).float()).sum().item()\n        else:\n            correct += (preds == labels).sum().item()\n        total_loss += loss.item() * images.size(0)\n        total += images.size(0)\n        pbar.set_postfix(loss=f\"{total_loss / total:.3f}\", acc=f\"{correct / total:.3f}\")\n    return total_loss / total, correct / total\n\n@torch.no_grad()\ndef evaluate(model, loader, criterion, desc=\"eval\"):\n    model.eval()\n    total_loss, correct, total = 0.0, 0, 0\n    for images, labels in tqdm(loader, desc=desc, leave=False):\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=USE_AMP):\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n        total_loss += loss.item() * images.size(0)\n        correct += (outputs.argmax(1) == labels).sum().item()\n        total += images.size(0)\n    return total_loss / total, correct / total\n\ndef run_training(train_positions, epochs, eval_sets=None, tag=\"\"):\n    \"\"\"Train a fresh CNN with AdamW + warmup/cosine LR + CutMix.\n    eval_sets: optional dict {name: positions}, evaluated after every epoch.\"\"\"\n    seed_everything(SEED)\n    train_loader = make_loader(train_positions, train=True)\n    eval_loaders = {name: make_loader(pos, train=False) for name, pos in (eval_sets or {}).items()}\n\n    model = CNN().to(device)\n    optimizer = make_optimizer(model)\n    steps_per_epoch = len(train_loader)\n    scheduler = make_scheduler(optimizer, epochs * steps_per_epoch, WARMUP_EPOCHS * steps_per_epoch)\n    scaler = torch.amp.GradScaler(\"cuda\", enabled=USE_AMP)\n    criterion = nn.CrossEntropyLoss(label_smoothing=LABEL_SMOOTHING)\n\n    history = []\n    for epoch in range(epochs):\n        t0 = time.time()\n        use_cutmix = epoch < epochs - CUTMIX_OFF_EPOCHS\n        tr_loss, tr_acc = train_one_epoch(model, train_loader, optimizer, scheduler, scaler, criterion,\n                                          use_cutmix, desc=f\"{tag} epoch {epoch + 1}/{epochs}\")\n        row = {\"epoch\": epoch + 1, \"lr\": optimizer.param_groups[0][\"lr\"],\n               \"train_loss\": tr_loss, \"train_acc\": tr_acc}\n        for name, loader in eval_loaders.items():\n            _, row[f\"{name}_acc\"] = evaluate(model, loader, criterion, desc=f\"{tag} eval {name}\")\n        history.append(row)\n\n        msg = (f\"{tag} epoch {epoch + 1:>3}/{epochs}: lr={row['lr']:.2e}  \"\n               f\"train_acc={tr_acc:.4f}{'' if use_cutmix else ' (no CutMix)'}\")\n        for name in eval_loaders:\n            msg += f\"  {name}_acc={row[f'{name}_acc']:.4f}\"\n        msg += f\"  ({time.time() - t0:.0f}s)\"\n        print(msg)\n    return model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:58.451334Z","iopub.execute_input":"2026-10-06T17:44:58.452066Z","iopub.status.idle":"2026-10-06T17:44:58.47576Z","shell.execute_reply.started":"2026-10-06T17:44:58.45203Z","shell.execute_reply":"2026-10-06T17:44:58.474729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# --- SMOKE TEST (delete this cell afterwards) ---\n_saved = (WARMUP_EPOCHS, CUTMIX_OFF_EPOCHS, CUTMIX_PROB)\nWARMUP_EPOCHS, CUTMIX_OFF_EPOCHS, CUTMIX_PROB = 1, 1, 1.0   # epoch 1: CutMix on every batch, epoch 2: off\nm, h = run_training(train_pos[:2048], 2, eval_sets={\"holdout\": val_pos[:1024]}, tag=\"[smoke]\")\nWARMUP_EPOCHS, CUTMIX_OFF_EPOCHS, CUTMIX_PROB = _saved\nprint(h)\ndel m","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:44:58.476783Z","iopub.execute_input":"2026-10-06T17:44:58.477063Z","iopub.status.idle":"2026-10-06T17:45:28.933069Z","shell.execute_reply.started":"2026-10-06T17:44:58.47704Z","shell.execute_reply":"2026-10-06T17:45:28.93169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holdout_history = None\nif RUN_HOLDOUT_CHECK:\n    print(f\"=== Holdout check: train on {len(train_pos)} images, score {len(val_pos)} holdout images ===\")\n    model, holdout_history = run_training(train_pos, EPOCHS, eval_sets={\"holdout\": val_pos},\n                                          tag=\"[holdout]\")\n    best = max(holdout_history, key=lambda h: h[\"holdout_acc\"])\n    print(f\"\\nFinal holdout acc: {holdout_history[-1]['holdout_acc']:.4f} \"\n          f\"(best {best['holdout_acc']:.4f} at epoch {best['epoch']})\")\n    del model\n    if device.type == \"cuda\":\n        torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:45:28.935564Z","iopub.execute_input":"2026-10-06T17:45:28.936506Z","iopub.status.idle":"2026-10-06T17:46:16.420728Z","shell.execute_reply.started":"2026-10-06T17:45:28.936467Z","shell.execute_reply":"2026-10-06T17:46:16.416423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LR schedule (warmup + cosine) and, if available, the holdout learning curve\nsteps_per_epoch = 50000 // BATCH_SIZE\ntotal_steps = EPOCHS * steps_per_epoch\nlrs = [LR * lr_factor(s, total_steps, WARMUP_EPOCHS * steps_per_epoch, MIN_LR_RATIO)\n       for s in range(total_steps)]\n\nn_plots = 2 if holdout_history is not None else 1\nfig, axes = plt.subplots(1, n_plots, figsize=(6 * n_plots, 4))\naxes = np.atleast_1d(axes)\n\naxes[0].plot(np.arange(total_steps) / steps_per_epoch, lrs)\naxes[0].axvline(WARMUP_EPOCHS, ls=\"--\", c=\"gray\", label=\"end of warmup\")\naxes[0].set_xlabel(\"epoch\"); axes[0].set_ylabel(\"learning rate\")\naxes[0].set_title(\"Linear warmup + cosine annealing\"); axes[0].legend(); axes[0].grid(alpha=0.3)\n\nif holdout_history is not None:\n    ep = [h[\"epoch\"] for h in holdout_history]\n    axes[1].plot(ep, [h[\"train_acc\"] for h in holdout_history], label=\"train acc (CutMix-weighted)\")\n    axes[1].plot(ep, [h[\"holdout_acc\"] for h in holdout_history], label=\"holdout acc\")\n    axes[1].axvline(EPOCHS - CUTMIX_OFF_EPOCHS, ls=\"--\", c=\"gray\", label=\"CutMix off\")\n    axes[1].set_xlabel(\"epoch\"); axes[1].set_ylabel(\"accuracy\")\n    axes[1].set_title(\"Holdout run\"); axes[1].legend(); axes[1].grid(alpha=0.3)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:46:16.422524Z","iopub.status.idle":"2026-10-06T17:46:16.423023Z","shell.execute_reply.started":"2026-10-06T17:46:16.422762Z","shell.execute_reply":"2026-10-06T17:46:16.422803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_pos = np.concatenate([train_pos, val_pos])     # train + val = all 50,000 images\nprint(\"Final training on\", len(full_pos), \"images for\", EPOCHS, \"epochs\")\n\nfinal_model, final_history = run_training(full_pos, EPOCHS, eval_sets=None, tag=\"[final]\")\n\ntorch.save(final_model.state_dict(), f\"{work}/final_model.pt\")\nprint(\"Saved weights to\", f\"{work}/final_model.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:46:16.428127Z","iopub.status.idle":"2026-10-06T17:46:16.428678Z","shell.execute_reply.started":"2026-10-06T17:46:16.428437Z","shell.execute_reply":"2026-10-06T17:46:16.428464Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"# --- Extract test.7z (skipped if already extracted) ---\nif len(glob.glob(f\"{TEST_IMG_DIR}/*.png\")) != 300000:\n    import py7zr\n    print(\"Extracting test.7z ...\")\n    with py7zr.SevenZipFile(f\"{base}/test.7z\", mode=\"r\") as z:\n        z.extractall(path=work)\nprint(\"test images:\", len(glob.glob(f\"{TEST_IMG_DIR}/*.png\")))\n\n# --- Load test images in the exact order of sampleSubmission.csv ---\nsample_sub = pd.read_csv(f\"{base}/sampleSubmission.csv\")\ntest_ids = sample_sub[\"id\"].to_numpy()\nX_test = load_images(test_ids, TEST_IMG_DIR, desc=\"loading test images\")\n\ntest_loader = DataLoader(ArrayDataset(X_test, labels=None, transform=eval_transform),\n                         batch_size=512, shuffle=False, num_workers=NUM_WORKERS,\n                         pin_memory=(device.type == \"cuda\"))\n\n# --- Predict with test-time augmentation (original + horizontal flip) ---\n@torch.no_grad()\ndef predict(model, loader, tta=True):\n    model.eval()\n    preds = []\n    for images in tqdm(loader, desc=\"inference\"):\n        images = images.to(device, non_blocking=True)\n        with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=USE_AMP):\n            probs = model(images).float().softmax(dim=1)\n            if tta:\n                probs += model(torch.flip(images, dims=[3])).float().softmax(dim=1)\n        preds.append(probs.argmax(dim=1).cpu())\n    return torch.cat(preds).numpy()\n\ntest_preds = predict(final_model, test_loader, tta=True)\n\n# --- Build and save submission.csv ---\nsubmission = pd.DataFrame({\"id\": test_ids, \"label\": np.array(classes)[test_preds]})\nassert submission.shape == sample_sub.shape\nassert (submission[\"id\"].to_numpy() == sample_sub[\"id\"].to_numpy()).all()\nsubmission.to_csv(f\"{work}/submission.csv\", index=False)\n\nprint(submission.head())\nprint(\"\\nPredicted class distribution:\")\nprint(submission[\"label\"].value_counts())\nprint(\"\\nSaved to\", f\"{work}/submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T17:46:16.429569Z","iopub.status.idle":"2026-10-06T17:46:16.429976Z","shell.execute_reply.started":"2026-10-06T17:46:16.429765Z","shell.execute_reply":"2026-10-06T17:46:16.429788Z"}},"outputs":[],"execution_count":null}]}