{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":""}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"8a3b5fb7-541e-45ab-8c29-b1309a733276","cell_type":"markdown","source":"# CIFAR-10 from scratch: custom ResNet-style ConvNet (random init)\n\n**Model** `Net` (designed from scratch, 17.5M params at width 96): 2-conv stem, then 3 stages of [3x3 conv-BN-ReLU, max-pool, residual block], concatenated global avg+max pooling, dropout, linear head. All weights random (Kaiming normal; last BN gamma of each residual block = 0). No pre-trained weights.\n\n**Training** AdamW (no weight decay on BN/bias), per-step linear warmup (3 epochs) then cosine annealing, batch 512, 100 epochs, CutMix (p=0.5) + random crop (reflect pad 4) + horizontal flip, label smoothing 0.1, EMA of weights (0.999), fp16 autocast. 4 models (different seeds) are trained on 45,000 images (5,000 held out) and averaged at inference with flip test-time augmentation.","metadata":{}},{"id":"dec0b509-cb9b-4b71-a60d-37adca60070f","cell_type":"code","source":"WORK = '/kaggle/working'\nimport os, subprocess, time, glob\nos.makedirs(f'{WORK}/runs', exist_ok=True)","metadata":{},"outputs":[],"execution_count":null},{"id":"5897c339-ef7f-46c8-89d7-45b486a183f4","cell_type":"code","source":"%%writefile train.py\n\"\"\"CIFAR-10 ConvNet trained from scratch (random init): AdamW + warmup/cosine LR + CutMix + EMA.\"\"\"\nimport argparse, copy, json, math, os, time\nimport numpy as np\nimport torch, torch.nn as nn, torch.nn.functional as F\n\nMEAN, STD = (0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616)\n\n\n# ------------------------------------------------------------------ model (designed from scratch)\ndef conv_bn(cin, cout):\n    return nn.Sequential(nn.Conv2d(cin, cout, 3, 1, 1, bias=False), nn.BatchNorm2d(cout), nn.ReLU(inplace=True))\n\n\nclass Block(nn.Module):\n    \"\"\"Basic residual block: relu(x + BN(conv(relu(BN(conv(x)))))). The last BN gamma starts at 0,\n    so every block begins as an identity map.\"\"\"\n    def __init__(self, c):\n        super().__init__()\n        self.c1, self.b1 = nn.Conv2d(c, c, 3, 1, 1, bias=False), nn.BatchNorm2d(c)\n        self.c2, self.b2 = nn.Conv2d(c, c, 3, 1, 1, bias=False), nn.BatchNorm2d(c)\n        nn.init.zeros_(self.b2.weight)\n\n    def forward(self, x):\n        out = F.relu(self.b1(self.c1(x)), inplace=True)\n        return F.relu(x + self.b2(self.c2(out)), inplace=True)\n\n\nclass Net(nn.Module):\n    \"\"\"2-conv stem -> 3 x [conv-BN-ReLU, max-pool, residual block] -> avg+max pooled features -> linear.\"\"\"\n    def __init__(self, width=64, num_classes=10, drop=0.2):\n        super().__init__()\n        w1, w2, w3 = width * 2, width * 4, width * 8\n        self.stem = nn.Sequential(conv_bn(3, width // 2), conv_bn(width // 2, width))        # 32x32\n        self.s1 = nn.Sequential(conv_bn(width, w1), nn.MaxPool2d(2), Block(w1))               # 16x16\n        self.s2 = nn.Sequential(conv_bn(w1, w2), nn.MaxPool2d(2), Block(w2))                  # 8x8\n        self.s3 = nn.Sequential(conv_bn(w2, w3), nn.MaxPool2d(2), Block(w3))                  # 4x4\n        self.drop, self.fc = nn.Dropout(drop), nn.Linear(2 * w3, num_classes)\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n\n    def forward(self, x):\n        x = self.s3(self.s2(self.s1(self.stem(x))))\n        x = torch.cat([F.adaptive_avg_pool2d(x, 1), F.adaptive_max_pool2d(x, 1)], 1).flatten(1)\n        return self.fc(self.drop(x))\n\n\n# ------------------------------------------------------------------ data + augmentation (all on GPU)\ndef normalize(x_uint8):\n    x = x_uint8.float().div_(255)\n    m = torch.tensor(MEAN, device=x.device).view(1, 3, 1, 1)\n    s = torch.tensor(STD, device=x.device).view(1, 3, 1, 1)\n    return (x - m) / s\n\n\ndef random_crop_flip(x, pad=4):\n    \"\"\"Random 32x32 crop from the reflect-padded image + random horizontal flip, per sample.\"\"\"\n    n, c, h, w = x.shape\n    dev = x.device\n    xp = F.pad(x, (pad,) * 4, mode='reflect')\n    dy = torch.randint(0, 2 * pad + 1, (n,), device=dev)\n    dx = torch.randint(0, 2 * pad + 1, (n,), device=dev)\n    flip = torch.rand(n, device=dev) < 0.5\n    ar = torch.arange(h, device=dev)\n    rows = (dy[:, None] + ar)[:, None, :, None]\n    cols = dx[:, None] + ar\n    cols = torch.where(flip[:, None], cols.flip(1), cols)[:, None, None, :]\n    bi = torch.arange(n, device=dev)[:, None, None, None]\n    ci = torch.arange(c, device=dev)[None, :, None, None]\n    return xp[bi, ci, rows, cols]\n\n\ndef cutmix(x, y, alpha=1.0):\n    \"\"\"Paste a random box from a shuffled copy of the batch; labels are mixed by the pasted area.\"\"\"\n    lam = np.random.beta(alpha, alpha)\n    perm = torch.randperm(x.size(0), device=x.device)\n    h, w = x.shape[2:]\n    r = math.sqrt(1.0 - lam)\n    bh, bw = int(h * r), int(w * r)\n    cy, cx = np.random.randint(h), np.random.randint(w)\n    y1, y2 = max(cy - bh // 2, 0), min(cy + bh // 2, h)\n    x1, x2 = max(cx - bw // 2, 0), min(cx + bw // 2, w)\n    x[:, :, y1:y2, x1:x2] = x[perm][:, :, y1:y2, x1:x2]\n    lam = 1.0 - (y2 - y1) * (x2 - x1) / (h * w)\n    return x, y, y[perm], lam\n\n\n# ------------------------------------------------------------------ schedule, EMA, evaluation\ndef lr_at(step, total, warm, peak, floor=1e-6):\n    \"\"\"Linear warmup, then cosine annealing to ~0 (updated every step).\"\"\"\n    if step < warm:\n        return peak * (step + 1) / warm\n    p = (step - warm) / max(1, total - warm)\n    return floor + 0.5 * (peak - floor) * (1 + math.cos(math.pi * p))\n\n\nclass EMA:\n    \"\"\"Exponential moving average of all weights and BN statistics.\"\"\"\n    def __init__(self, model, decay):\n        self.decay = decay\n        self.shadow = copy.deepcopy(model).eval()\n        for p in self.shadow.parameters():\n            p.requires_grad_(False)\n\n    @torch.no_grad()\n    def update(self, model, step):\n        d = min(self.decay, (1 + step) / (10 + step))\n        for s, m in zip(self.shadow.state_dict().values(), model.state_dict().values()):\n            if s.dtype.is_floating_point:\n                s.mul_(d).add_(m.detach(), alpha=1 - d)\n            else:\n                s.copy_(m)\n\n\n@torch.no_grad()\ndef predict(model, X, bs=1000, tta=False):\n    \"\"\"Class probabilities for uint8 NCHW images; tta=True averages the image and its mirror.\"\"\"\n    model.eval()\n    out = []\n    with torch.autocast('cuda', dtype=torch.float16):\n        for i in range(0, len(X), bs):\n            x = normalize(X[i:i + bs]).contiguous(memory_format=torch.channels_last)\n            p = model(x).float().softmax(1)\n            if tta:\n                p = p + model(x.flip(3)).float().softmax(1)\n            out.append(p)\n    return torch.cat(out)\n\n\ndef accuracy(model, X, Y, tta=False):\n    return (predict(model, X, tta=tta).argmax(1) == Y).float().mean().item() * 100\n\n\n# ------------------------------------------------------------------ main\ndef main():\n    ap = argparse.ArgumentParser()\n    ap.add_argument('--name', default='run')\n    ap.add_argument('--epochs', type=int, default=100)\n    ap.add_argument('--bs', type=int, default=512)\n    ap.add_argument('--lr', type=float, default=2e-3)\n    ap.add_argument('--wd', type=float, default=0.05)\n    ap.add_argument('--warmup', type=float, default=3, help='warmup length in epochs')\n    ap.add_argument('--cutmix_p', type=float, default=0.5)\n    ap.add_argument('--ls', type=float, default=0.1)\n    ap.add_argument('--ema', type=float, default=0.999)\n    ap.add_argument('--width', type=int, default=64)\n    ap.add_argument('--drop', type=float, default=0.2)\n    ap.add_argument('--seed', type=int, default=0)\n    ap.add_argument('--holdout', type=int, default=5000, help='validation images held out of training (0 = train on all)')\n    ap.add_argument('--max_train', type=int, default=0, help='use only the first N training images (smoke tests)')\n    ap.add_argument('--baseline', default='none')\n    ap.add_argument('--changed', default='first run')\n    ap.add_argument('--work', default='/kaggle/working')\n    a = ap.parse_args()\n\n    out_dir = f'{a.work}/runs/{a.name}'\n    os.makedirs(out_dir, exist_ok=True)\n    logf = open(f'{out_dir}/log.txt', 'a')\n\n    def log(s):\n        print(s, flush=True)\n        logf.write(s + '\\n')\n        logf.flush()\n\n    torch.manual_seed(a.seed)\n    np.random.seed(a.seed)\n    torch.backends.cudnn.benchmark = True\n    dev = torch.device('cuda')\n\n    X = np.load(f'{a.work}/train_x.npy')                       # (50000, 32, 32, 3) uint8\n    Y = np.load(f'{a.work}/train_y.npy')\n    if a.max_train:\n        X, Y = X[:a.max_train], Y[:a.max_train]\n    split = np.random.RandomState(1234).permutation(len(X))    # same split for every run\n    va, tr = split[:a.holdout], split[a.holdout:]\n    to_gpu = lambda arr: torch.from_numpy(arr).to(dev)\n    Xtr, Ytr = to_gpu(X[tr]).permute(0, 3, 1, 2).contiguous(), to_gpu(Y[tr])\n    Xva, Yva = to_gpu(X[va]).permute(0, 3, 1, 2).contiguous(), to_gpu(Y[va])\n\n    model = Net(a.width, 10, a.drop).to(dev).to(memory_format=torch.channels_last)\n    n_params = sum(p.numel() for p in model.parameters())\n    decay = [p for p in model.parameters() if p.ndim > 1]\n    no_decay = [p for p in model.parameters() if p.ndim <= 1]\n    opt = torch.optim.AdamW([{'params': decay, 'weight_decay': a.wd}, {'params': no_decay, 'weight_decay': 0.0}],\n                            lr=a.lr, betas=(0.9, 0.999))\n    scaler = torch.amp.GradScaler('cuda')\n    ema = EMA(model, a.ema)\n\n    n = len(Xtr)\n    steps_per_epoch = n // a.bs\n    total, warm = a.epochs * steps_per_epoch, int(a.warmup * steps_per_epoch)\n    log(f'{a.name}: {n} train / {len(Xva)} val, {n_params / 1e6:.2f}M params, {steps_per_epoch} steps/epoch, args={vars(a)}')\n\n    step, t_start = 0, time.time()\n    for ep in range(a.epochs):\n        model.train()\n        t0 = time.time()\n        run_loss = torch.zeros((), device=dev)\n        perm = torch.randperm(n, device=dev)\n        for k in range(steps_per_epoch):\n            idx = perm[k * a.bs:(k + 1) * a.bs]\n            x, y = random_crop_flip(normalize(Xtr[idx])), Ytr[idx]\n            use_cm = np.random.rand() < a.cutmix_p\n            if use_cm:\n                x, ya, yb, lam = cutmix(x, y)\n            x = x.contiguous(memory_format=torch.channels_last)\n            lr = lr_at(step, total, warm, a.lr)\n            for g in opt.param_groups:\n                g['lr'] = lr\n            with torch.autocast('cuda', dtype=torch.float16):\n                logits = model(x)\n            logits = logits.float()\n            if use_cm:\n                loss = lam * F.cross_entropy(logits, ya, label_smoothing=a.ls) + (1 - lam) * F.cross_entropy(logits, yb, label_smoothing=a.ls)\n            else:\n                loss = F.cross_entropy(logits, y, label_smoothing=a.ls)\n            opt.zero_grad(set_to_none=True)\n            scaler.scale(loss).backward()\n            scaler.step(opt)\n            scaler.update()\n            ema.update(model, step)\n            run_loss += loss.detach()\n            step += 1\n        msg = f'ep {ep + 1:3d}/{a.epochs} loss {run_loss.item() / steps_per_epoch:.3f} lr {lr:.2e} epoch_time {time.time() - t0:.1f}s'\n        if len(Xva):\n            msg += f' | val raw {accuracy(model, Xva, Yva):.2f} ema {accuracy(ema.shadow, Xva, Yva):.2f}'\n        log(msg)\n\n    elapsed = time.time() - t_start\n    rec = dict(task='CIFAR-10 classification (Kaggle late submission)',\n               split=f'{n} train / {len(Xva)} held-out validation (split seed 1234)',\n               baseline=a.baseline, model=f'Net(width={a.width}), {n_params / 1e6:.2f}M params, random init',\n               changed_setting=a.changed, seed=a.seed, hardware=torch.cuda.get_device_name(0),\n               optimizer=f'AdamW lr={a.lr} wd={a.wd}, {a.warmup} warmup epochs then cosine, bs={a.bs}',\n               augmentation=f'random crop(pad 4)+flip, CutMix p={a.cutmix_p}, label smoothing {a.ls}, EMA {a.ema}',\n               elapsed_s=round(elapsed), artifact=f'{out_dir}/model.pt',\n               final_test_rule='Kaggle test set is evaluated once, after the recipe is frozen')\n    if len(Xva):\n        res = {'raw': accuracy(model, Xva, Yva), 'raw_tta': accuracy(model, Xva, Yva, tta=True),\n               'ema': accuracy(ema.shadow, Xva, Yva), 'ema_tta': accuracy(ema.shadow, Xva, Yva, tta=True)}\n        rec['val_accuracy'] = {k: round(v, 2) for k, v in res.items()}\n        best = max(res, key=res.get)\n        rec['best_variant'] = best\n        m = ema.shadow if best.startswith('ema') else model\n        pred = predict(m, Xva, tta=best.endswith('tta')).argmax(1)\n        conf = torch.zeros(10, 10, dtype=torch.long)\n        for t, p in zip(Yva.cpu(), pred.cpu()):\n            conf[t, p] += 1\n        rec['confusion'] = conf.tolist()\n        log(f'final val accuracy: {rec[\"val_accuracy\"]}')\n    torch.save({'model': model.state_dict(), 'ema': ema.shadow.state_dict(), 'args': vars(a)}, f'{out_dir}/model.pt')\n    json.dump(rec, open(f'{out_dir}/record.json', 'w'), indent=1)\n    log(f'done in {elapsed / 60:.1f} min -> {out_dir}')\n\n\nif __name__ == '__main__':\n    main()\n","metadata":{},"outputs":[],"execution_count":null},{"id":"b221453a-a76a-4ae0-ba96-72aa92eb5090","cell_type":"code","source":"%%writefile prep.py\n\"\"\"Read CIFAR-10 PNGs straight out of a .7z archive (no per-file disk writes) into a uint8 array.\nusage: python prep.py train|test\"\"\"\nimport io, re, subprocess, sys, time, os\nimport numpy as np, pandas as pd\nfrom PIL import Image\nfrom concurrent.futures import ThreadPoolExecutor\n\nCOMP = '/kaggle/input/competitions/cifar-10'\nIEND = b'IEND\\xaeB`\\x82'\n\n\ndef split_pngs(data):\n    \"\"\"Cut a byte stream of concatenated PNG files into the individual files.\"\"\"\n    out, start = [], 0\n    for m in re.finditer(re.escape(IEND), data):\n        out.append(data[start:m.end()])\n        start = m.end()\n    return out\n\n\ndef read_archive(arc):\n    t = time.time()\n    lst = subprocess.run(['7z', 'l', '-slt', arc], capture_output=True, text=True).stdout.splitlines()\n    body = lst[lst.index('----------') + 1:]\n    names = [l[7:] for l in body if l.startswith('Path = ') and l.lower().endswith('.png')]\n    print(f'{arc}: {len(names)} png entries listed in {time.time() - t:.0f}s', flush=True)\n    p = subprocess.Popen(['7z', 'x', '-so', '-y', arc], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)\n    data = p.stdout.read()\n    p.wait()\n    files = split_pngs(data)\n    print(f'streamed {len(data) / 1e6:.0f} MB, {len(files)} pngs in {time.time() - t:.0f}s', flush=True)\n    assert len(files) == len(names), (len(files), len(names))\n    return {int(os.path.basename(n).split('.')[0]): f for n, f in zip(names, files)}\n\n\ndef decode(files, ids, workers=4):\n    def rd(i):\n        return np.asarray(Image.open(io.BytesIO(files[i])).convert('RGB'))\n    with ThreadPoolExecutor(workers) as ex:\n        return np.stack(list(ex.map(rd, ids)))\n\n\nif __name__ == '__main__':\n    which = sys.argv[1]\n    if which == 'train':\n        lab = pd.read_csv(f'{COMP}/trainLabels.csv')\n        classes = sorted(lab['label'].unique())\n        files = read_archive(f'{COMP}/train.7z')\n        X = decode(files, lab['id'].tolist())\n        y = lab['label'].map({c: i for i, c in enumerate(classes)}).values.astype(np.int64)\n        np.save('/kaggle/working/train_x.npy', X)\n        np.save('/kaggle/working/train_y.npy', y)\n        print('train', X.shape, np.bincount(y), classes, flush=True)\n    else:\n        ids = pd.read_csv(f'{COMP}/sampleSubmission.csv')['id'].tolist()\n        files = read_archive(f'{COMP}/test.7z')\n        t = time.time()\n        X = decode(files, ids)\n        print('test', X.shape, f'decoded in {time.time() - t:.0f}s', flush=True)\n        np.save('/tmp/test_x.npy', X)\n        open('/tmp/test_x.done', 'w').write('ok')\n        print('saved /tmp/test_x.npy', flush=True)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"1a8336f2-2038-4e7d-82b1-998e4297ddde","cell_type":"code","source":"%%writefile infer.py\n\"\"\"Ensemble the trained models (EMA weights, flip TTA) and write submission.csv for all test images.\"\"\"\nimport glob, numpy as np, pandas as pd, torch\nimport train\n\nCOMP = '/kaggle/input/competitions/cifar-10'\nW = '/kaggle/working'\ndev = torch.device('cuda')\nto_t = lambda a: torch.from_numpy(a).to(dev).permute(0, 3, 1, 2).contiguous()\n\nlab = pd.read_csv(f'{COMP}/trainLabels.csv')\nclasses = sorted(lab['label'].unique())\nXtr, Ytr = np.load(f'{W}/train_x.npy'), np.load(f'{W}/train_y.npy')\nva = np.random.RandomState(1234).permutation(len(Xtr))[:5000]       # same held-out split as training\nXva, Yva = to_t(Xtr[va]), torch.from_numpy(Ytr[va]).to(dev)\nXte = to_t(np.load('/tmp/test_x.npy'))\nprint('test images', tuple(Xte.shape))\n\nvsum = tsum = 0\nruns = sorted(glob.glob(f'{W}/runs/final_*/model.pt'))\nfor r in runs:\n    ck = torch.load(r, map_location=dev)\n    a = ck['args']\n    m = train.Net(a['width'], 10, a['drop']).to(dev).to(memory_format=torch.channels_last)\n    m.load_state_dict(ck['ema'])\n    vp = train.predict(m, Xva, tta=True)\n    print(r.split('/')[-2], 'held-out acc %.2f' % ((vp.argmax(1) == Yva).float().mean().item() * 100), flush=True)\n    vsum, tsum = vsum + vp, tsum + train.predict(m, Xte, tta=True)\nprint('ENSEMBLE of', len(runs), 'models: held-out acc %.2f' % ((vsum.argmax(1) == Yva).float().mean().item() * 100))\n\npred = tsum.argmax(1).cpu().numpy()\nids = pd.read_csv(f'{COMP}/sampleSubmission.csv')['id']\nsub = pd.DataFrame({'id': ids, 'label': [classes[i] for i in pred]})\nsub.to_csv(f'{W}/submission.csv', index=False)\nprint(sub.shape); print(sub.head()); print(sub['label'].value_counts())\n","metadata":{},"outputs":[],"execution_count":null},{"id":"521d33c2-1082-48c5-9f82-18ca37c77d43","cell_type":"markdown","source":"## 1. Data: read the PNGs straight from the .7z archives into uint8 arrays\nThe 300,000 test images are prepared in the background while the models train.","metadata":{}},{"id":"fb884961-52cd-4a43-b7f4-c6074d3be819","cell_type":"code","source":"test_prep = subprocess.Popen('python prep.py test > /tmp/prep_test.log 2>&1', shell=True, cwd=WORK)\n!python prep.py train","metadata":{},"outputs":[],"execution_count":null},{"id":"51fb6375-7e5c-44c8-8c4d-d6c93d9b4c79","cell_type":"markdown","source":"## 2. Train 4 models from random init (2 per GPU, two GPUs in parallel)","metadata":{}},{"id":"9f0be302-afab-4e4d-9c3d-dc5632cc2705","cell_type":"code","source":"SEEDS = {0: [0, 2], 1: [1, 3]}          # GPU id -> seeds trained one after the other\nprocs = []\nfor gpu, seeds in SEEDS.items():\n    cmd = ' ; '.join(f'python train.py --name final_s{s} --epochs 100 --width 96 --seed {s} '\n                     f'--baseline dev_w96 --changed \"final recipe, seed {s}\" > runs/final_s{s}.out 2>&1' for s in seeds)\n    procs.append(subprocess.Popen(cmd, shell=True, cwd=WORK, env=dict(os.environ, CUDA_VISIBLE_DEVICES=str(gpu))))\nwhile any(p.poll() is None for p in procs):\n    time.sleep(300)\n    for f in sorted(glob.glob(f'{WORK}/runs/final_s*.out')):\n        print(os.path.basename(f), open(f).read().strip().split('\\n')[-1][:130], flush=True)\nprint('training finished')","metadata":{},"outputs":[],"execution_count":null},{"id":"8816e5df-32ec-42a5-ac4b-aab0a6aeef13","cell_type":"code","source":"for f in sorted(glob.glob(f'{WORK}/runs/final_s*/record.json')):\n    import json; r = json.load(open(f)); print(f.split('/')[-2], r['model'], r['val_accuracy'], f\"{r['elapsed_s']/60:.1f} min\")","metadata":{},"outputs":[],"execution_count":null},{"id":"8d681e38-a060-4fda-b255-04e2619555a3","cell_type":"markdown","source":"## 3. Predict all 300,000 test images (ensemble, flip TTA) and write `submission.csv`","metadata":{}},{"id":"6ce20249-2019-4500-af09-b581a241a622","cell_type":"code","source":"while not os.path.exists('/tmp/test_x.done') and test_prep.poll() is None:\n    time.sleep(30)\nassert os.path.exists('/tmp/test_x.done'), open('/tmp/prep_test.log').read()[-800:]\nprint(open('/tmp/prep_test.log').read()[-400:])\n!python infer.py","metadata":{},"outputs":[],"execution_count":null},{"id":"02c337d9-1fab-4fc9-9164-f7c67a60c026","cell_type":"code","source":"import pandas as pd\ns = pd.read_csv(f'{WORK}/submission.csv'); print(s.shape, s['id'].min(), s['id'].max()); s.head()","metadata":{},"outputs":[],"execution_count":null}]}