{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11"},"kaggle":{"accelerator":"nvidiaTeslaT4","isInternetEnabled":true,"isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"de4db711-2436-45c6-b97e-25d466a9da9d","cell_type":"markdown","source":"# Round 13 · 02 — ConvNeXt-Base (ImageNet-22k) @ 512\n\n**Goal:** honest 5-class APTOS test accuracy > **89.70 %** (previous honest best 86.58 %).\n\nA modern CNN with ImageNet-22k pretraining. It was never actually run in round 9 or 11, and it adds diversity for the final ensemble. AdamW with lr 1e-4 and wd 0.05, drop-path 0.3.\n\n**Inputs to attach (`+ Add Input`):** APTOS 2019 competition data, `naitik18/idrid-dataset`, `mariaherrerot/messidor2preprocess`.\n**Settings:** Accelerator **GPU T4 x2** or P100 (training uses one GPU), Internet **ON**, Persistence off. Use *Save Version → Save & Run All (Commit)*.\n\n**Outputs:** `/kaggle/working/preds/<run_name>/` (val/test/unlabelled probabilities + `keys.csv` + `meta.json`). Notebooks 08 and 10 read these.\n\n**Knobs:** everything is in the CONFIG cell. Lines marked `<-- changed` differ from the shared defaults. `SMOKE=True` runs a tiny synthetic CPU dry-run.\n\n**Runtime:** the notebook plans itself to finish inside `BUDGET_HOURS` (10.3 h, under Kaggle's 12 h limit). Seeds are skipped and epochs shortened only if time runs short; the log says so (`budget -> training N epochs`). If you see that, lower `N_RUNS`.\n\nThe test set is reported **once**, at the end. Every choice (checkpoint, seed averaging, decoder) is made on val.","metadata":{}},{"id":"3a9665a8-6ff6-4e95-a132-d9f4add69c14","cell_type":"code","source":"# ============================== CONFIG ==============================\nSMOKE = False      # True = tiny synthetic CPU dry-run of the whole pipeline (local testing only)\nRUN_NAME = 'r13_02_convnext_base_512'\n\nclass CFG:\n    SEED = 42   # split seed: MUST stay 42 to reproduce the 708-image test set of rounds 5-10\n    NUM_CLASSES = 5\n    CLASS_NAMES = ['No-DR', 'Mild', 'Moderate', 'Severe', 'PDR']\n    TEST_FRAC = 0.2   # APTOS-only stratified hold-out (same as rounds 5-10)\n    VAL_FRAC = 0.1\n    EXTRA_GRADES = (1, 2, 3, 4)   # IDRiD / Messidor-2 grades added to TRAIN (never to val/test)\n    MD5_DEDUPE = True   # exact-byte dedup before the split (keeps the round-10 split identical)\n    PURGE_NEAR_DUPS = True   # drop TRAIN/unlabelled images that are near-duplicates (dHash) of val/test images\n    DHASH_HAMMING = 8   # 256-bit dHash distance that counts as 'same photo'\n    REQUIRE_EXTERNAL = True   # hard stop if IDRiD or Messidor-2 is not attached\n    CACHE_SIZE = 512   # preprocessed uint8 cache resolution (fundus crop + LAB-L CLAHE)\n    CLAHE_CLIP = 2.0\n    CLAHE_GRID = 8\n    CROP_THRESHOLD = 7\n    IMG_SIZE = 512   # <-- changed for this notebook\n    MODEL_NAMES = ['convnext_base.fb_in22k_ft_in1k_384', 'convnext_base.fb_in22k_ft_in1k', 'convnext_small.fb_in22k_ft_in1k_384']   # <-- changed for this notebook\n    MODEL_KW = {}   # extra timm.create_model kwargs\n    PRETRAINED = True\n    DROP = 0.3   # dropout before the head\n    DROP_PATH = 0.3   # <-- changed for this notebook\n    MULTITASK = False   # extra regression + ordinal heads\n    W_REG = 0.5\n    W_ORD = 0.5\n    EPOCHS = 20   # <-- changed for this notebook\n    WARMUP_EPOCHS = 1\n    BATCH_SIZE = 8   # batch per step\n    ACCUM = 4   # gradient accumulation -> effective batch BATCH_SIZE*ACCUM\n    MULTI_GPU = False   # nn.DataParallel over both T4s. OFF: it crashed the kernel on Kaggle T4x2 (c10::AcceleratorError)\n    CHANNELS_LAST = False   # memory-format speed-up; off for stability\n    LR = 0.0001   # <-- changed for this notebook\n    HEAD_LR_MULT = 10.0\n    WD = 0.05   # <-- changed for this notebook\n    LABEL_SMOOTH = 0.05\n    SAMPLER_POWER = 0.3   # sample weight (1/class_count)^power; 0 = plain shuffle\n    MIXUP_P = 0.0   # probability a batch is mixed (mixup)\n    MIXUP_ALPHA = 0.4\n    STRONG_AUG = False   # heavier colour/scale jitter (noisy student)\n    EMA_DECAY = 0.998   # with warm-up ramp; each epoch keeps whichever of raw/EMA scores best on val\n    GRAD_CLIP = 1.0\n    SELECT_W_ACC = 0.8   # checkpoint score = w*acc + (1-w)*QWK on val\n    N_RUNS = 2   # <-- changed for this notebook\n    TTA = 8   # 1, 2, 4 or 8 views (flips x rot90)\n    BUDGET_HOURS = 10.3   # wall-clock budget for the whole notebook (Kaggle limit is 12h)\n    RESERVE_MIN = 30   # minutes kept for inference + reporting\n    DECODER_GUARD = 0.5   # class-bias decoder adopted only if it beats argmax by >= this many val points\n    NUM_WORKERS = 4\n    PREDICT_UNL = True   # also predict the 1,928 unlabelled APTOS test images (needed by notebooks 07/08)\n    SAVE_WEIGHTS = False   # save best weights of run 0 to /kaggle/working (large files)\n    TARGET = 89.7   # goal accuracy (%)\n\ncfg = CFG()\nif SMOKE:\n    cfg.CACHE_SIZE = 96\n    cfg.IMG_SIZE = 96\n    cfg.PRETRAINED = False\n    cfg.EPOCHS = 2\n    cfg.N_RUNS = 2\n    cfg.BATCH_SIZE = 8\n    cfg.ACCUM = 1\n    cfg.TTA = 2\n    cfg.NUM_WORKERS = 0\n    cfg.BUDGET_HOURS = 2.0\n    cfg.RESERVE_MIN = 1\n    cfg.REQUIRE_EXTERNAL = True","metadata":{},"outputs":[],"execution_count":null},{"id":"3aeb151d-80ad-4ecc-9048-1eefca467024","cell_type":"code","source":"import os, sys, gc, json, time, math, copy, random, hashlib, zipfile, warnings\nfrom collections import Counter, defaultdict\nimport numpy as np, pandas as pd, cv2\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score, cohen_kappa_score, f1_score, confusion_matrix\nwarnings.filterwarnings(\"ignore\")\ncv2.setNumThreads(0)\nT_START = time.time()\n\nimport torch, torch.nn as nn, torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nimport timm\ntorch.backends.cudnn.benchmark = True\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nN_GPU = torch.cuda.device_count()\nUSE_AMP = device.type == \"cuda\"\nprint(\"torch\", torch.__version__, \"| timm\", timm.__version__, \"| device\", device, \"| GPUs\", N_GPU)\nfor i in range(N_GPU): print(\"  GPU\", i, torch.cuda.get_device_name(i))\nif device.type == \"cuda\": torch.cuda.set_device(0)\nMF = torch.channels_last if cfg.CHANNELS_LAST else torch.contiguous_format\n\ndef set_seed(s):\n    random.seed(s); np.random.seed(s); torch.manual_seed(s); torch.cuda.manual_seed_all(s)\nset_seed(cfg.SEED)\n\ndef hours_left():\n    return cfg.BUDGET_HOURS - (time.time() - T_START) / 3600\n\nWORK = \"/kaggle/working\" if os.path.isdir(\"/kaggle/working\") else os.path.abspath(\"./working\")\nOUT_DIR = os.path.join(WORK, \"preds\")\nos.makedirs(OUT_DIR, exist_ok=True)\nCACHE_DIR = \"/tmp/r13_cache\"\nos.makedirs(CACHE_DIR, exist_ok=True)\nprint(\"RUN\", RUN_NAME, \"| outputs ->\", OUT_DIR)","metadata":{},"outputs":[],"execution_count":null},{"id":"1b1ccb12-5c01-4bd7-b2bf-7f3c87ff6879","cell_type":"code","source":"# Builds a tiny fake copy of the three datasets (same folder/csv layout as Kaggle) when SMOKE=True.\nINPUT_ROOT = \"/kaggle/input\"\nif SMOKE:\n    INPUT_ROOT = \"/tmp/r13_smoke_input\"\n    if not os.path.isdir(INPUT_ROOT):\n        rng = np.random.default_rng(0)\n        def fake(grade, size):\n            img = np.zeros((size, int(size * 1.3), 3), np.uint8)\n            c = (img.shape[1] // 2, size // 2)\n            cv2.circle(img, c, size // 2 - 2, (150, 80, 40), -1)\n            for _ in range(grade * 6):          # grade-dependent \"lesions\" so the net can learn something\n                p = (int(c[0] + rng.integers(-size // 3, size // 3)), int(c[1] + rng.integers(-size // 3, size // 3)))\n                cv2.circle(img, p, int(rng.integers(2, 5)), (30, 30, 200), -1)\n            return np.clip(img.astype(int) + rng.integers(0, 12, img.shape), 0, 255).astype(np.uint8)\n        def mk(folder, names, grades, ext=\".png\"):\n            os.makedirs(folder, exist_ok=True)\n            for n, g in zip(names, grades): cv2.imwrite(os.path.join(folder, n + ext), fake(g, 120))\n        ap = os.path.join(INPUT_ROOT, \"competitions/aptos2019-blindness-detection\")\n        g = np.repeat(np.arange(5), [70, 30, 50, 25, 25]); ids = [f\"ap{i:04d}\" for i in range(len(g))]\n        mk(ap + \"/train_images\", ids, g)\n        mk(ap + \"/train_images\", [\"apdup0000\"], [g[0]])            # an exact duplicate (md5)\n        import shutil; shutil.copy(ap + \"/train_images/ap0000.png\", ap + \"/train_images/apdup0000.png\")\n        pd.DataFrame({\"id_code\": ids + [\"apdup0000\"], \"diagnosis\": list(g) + [g[0]]}).to_csv(ap + \"/train.csv\", index=False)\n        uid = [f\"te{i:04d}\" for i in range(20)]\n        mk(ap + \"/test_images\", uid, rng.integers(0, 5, 20))\n        pd.DataFrame({\"id_code\": uid}).to_csv(ap + \"/test.csv\", index=False)\n        pd.DataFrame({\"id_code\": uid, \"diagnosis\": 0}).to_csv(ap + \"/sample_submission.csv\", index=False)\n        idr = os.path.join(INPUT_ROOT, \"datasets/naitik18/idrid-dataset/idrid\")\n        gi = np.repeat(np.arange(5), 6); ni = [f\"IDRiD_{i:03d}\" for i in range(len(gi))]\n        mk(idr + \"/images\", ni, gi, \".jpg\")\n        pd.DataFrame({\"Image name\": ni, \"Retinopathy grade\": gi, \"Risk of macular edema \": 0}).to_csv(\n            idr + \"/a. IDRiD_Disease Grading_Training Labels.csv\", index=False)\n        ms = os.path.join(INPUT_ROOT, \"datasets/mariaherrerot/messidor2preprocess\")\n        gm = np.repeat(np.arange(5), 6); nm = [f\"20051020_{i:04d}_0100_PP\" for i in range(len(gm))]\n        mk(ms + \"/messidor-2/messidor-2/preprocess\", nm, gm)\n        pd.DataFrame({\"id_code\": [n + \".png\" for n in nm], \"diagnosis\": gm, \"adjudicated_dme\": 0,\n                      \"adjudicated_gradable\": 1}).to_csv(ms + \"/messidor_data.csv\", index=False)\n    print(\"SMOKE input at\", INPUT_ROOT)","metadata":{},"outputs":[],"execution_count":null},{"id":"bd94e8f9-3d24-4cef-9734-3d05f9abad96","cell_type":"code","source":"# ---- dataset discovery: round-10 loaders (keeps the exact same row order -> same split) ----\ndef _list_files(root, exts):\n    out = []\n    for dp, _, fns in os.walk(root, followlinks=True):\n        for fn in fns:\n            if fn.lower().endswith(exts): out.append(os.path.join(dp, fn))\n    return out\n\nall_csvs   = _list_files(INPUT_ROOT, (\".csv\",))\nall_images = _list_files(INPUT_ROOT, (\".png\", \".jpg\", \".jpeg\", \".tif\", \".tiff\"))\nall_zips   = _list_files(INPUT_ROOT, (\".zip\",))\nprint(f\"Scanned {INPUT_ROOT}: {len(all_csvs)} csv, {len(all_images)} images, {len(all_zips)} zips\")\nfor p in all_csvs: print(\"   csv:\", p)\n\ndef build_image_index(paths):\n    idx = {}\n    for p in paths: idx.setdefault(os.path.splitext(os.path.basename(p))[0].lower(), p)\n    return idx\nIMG_INDEX = build_image_index(all_images)\nIMG_FULL  = {os.path.basename(p).lower(): p for p in all_images}\n\n# ---- APTOS (labelled) ----\nframes, srcs = [], []\nfor p in all_csvs:\n    if any(k in p.lower() for k in [\"messidor\", \"idrid\"]): continue\n    try: df = pd.read_csv(p)\n    except Exception: continue\n    df.columns = [c.strip().lower() for c in df.columns]\n    if not (any(\"id_code\" in c for c in df.columns) and any(\"diagnosis\" in c for c in df.columns)): continue\n    ic = [c for c in df.columns if \"id_code\" in c][0]\n    dc = [c for c in df.columns if \"diagnosis\" in c][0]\n    sub = df[[ic, dc]].dropna(); sub.columns = [\"image_id\", \"grade\"]\n    if sub[\"grade\"].nunique() < 2: continue            # unlabelled submission file\n    frames.append(sub); srcs.append((p, len(sub)))\nassert frames, \"APTOS labels not found (need id_code + diagnosis). Attach the APTOS 2019 competition data.\"\nfor p, n in srcs: print(f\"APTOS labels: {n} rows <- {p}\")\naptos_df = pd.concat(frames, ignore_index=True)\naptos_df = aptos_df.drop_duplicates(subset=[\"image_id\"], keep=\"first\").reset_index(drop=True)\naptos_df[\"grade\"] = aptos_df[\"grade\"].astype(int)\naptos_csv_set = {p for p, _ in srcs}\ndef _res_aptos(i):\n    s = str(i).strip().lower()\n    return (IMG_INDEX.get(s) or IMG_FULL.get(s) or IMG_FULL.get(s + \".png\")\n            or IMG_FULL.get(s + \".jpg\") or IMG_FULL.get(s + \".jpeg\"))\naptos_df[\"path\"] = aptos_df[\"image_id\"].map(_res_aptos)\naptos_df = aptos_df.dropna(subset=[\"path\"]).reset_index(drop=True)\naptos_df[\"source\"] = \"APTOS\"\nprint(\"APTOS usable:\", len(aptos_df), \"| grades\", dict(sorted(Counter(aptos_df[\"grade\"]).items())))\n\ndef _pick(cols, keys, bad=()):\n    low = {c.lower().strip(): c for c in cols}\n    for k in keys:\n        for lc, orig in low.items():\n            if (k == lc or k in lc) and not any(b in lc for b in bad): return orig\n    return None\n\ndef _unzip_matching(word):\n    for z in all_zips:\n        try:\n            with zipfile.ZipFile(z) as zf:\n                if any(word in n.lower() for n in zf.namelist()):\n                    dest = f\"/tmp/{word}_imgs\"; os.makedirs(dest, exist_ok=True); zf.extractall(dest)\n                    new = _list_files(dest, (\".png\", \".jpg\", \".jpeg\", \".tif\", \".tiff\")); all_images.extend(new)\n                    IMG_FULL.update({os.path.basename(q).lower(): q for q in new})\n                    IMG_INDEX.update(build_image_index(new)); print(\"  extracted\", z)\n        except Exception as e: print(\"  zip error:\", e)\n\ndef load_idrid():\n    NAME = [\"image name\", \"image_name\", \"imagename\", \"image\", \"file\", \"filename\", \"name\"]\n    GRAD = [\"retinopathy grade\", \"retinopathy_grade\", \"grade\", \"diagnosis\", \"dr_grade\", \"class\", \"label\"]\n    hit = None\n    for p in all_csvs:\n        if \"idrid\" not in p.lower() and \"disease grading\" not in os.path.basename(p).lower(): continue\n        try: df = pd.read_csv(p)\n        except Exception: continue\n        nc, gc = _pick(df.columns, NAME), _pick(df.columns, GRAD)\n        if nc and gc: hit = (p, df, nc, gc); break\n    if hit is None: print(\"IDRiD: no matching csv.\"); return None\n    p, df, nc, gc = hit\n    df = df[[nc, gc]].dropna(); df.columns = [\"image_id\", \"grade\"]\n    df[\"grade\"] = df[\"grade\"].astype(float).round().astype(int)\n    df = df[df[\"grade\"].between(0, 4)]\n    if not any(str(n).strip().lower() in IMG_INDEX for n in df[\"image_id\"].head(20)): _unzip_matching(\"idrid\")\n    df[\"path\"] = df[\"image_id\"].map(lambda n: IMG_INDEX.get(str(n).strip().lower()))\n    df = df.dropna(subset=[\"path\"]).reset_index(drop=True); df[\"source\"] = \"IDRiD\"\n    print(f\"IDRiD csv: {p} | usable {len(df)} | grades {dict(sorted(Counter(df['grade']).items()))}\")\n    return df if len(df) else None\n\n_MG = [\"adjudicated_dr_grade\", \"dr_grade\", \"retinopathy grade\", \"retinopathy_grade\",\n       \"dr_level\", \"diagnosis\", \"grade\", \"level\", \"label\", \"class\", \"severity\"]\n_MI = [\"image_id\", \"image id\", \"image name\", \"image_name\", \"imagename\", \"image\",\n       \"id_code\", \"filename\", \"file_name\", \"file\", \"img\", \"name\", \"id\"]\ndef _resolve_any(n):\n    s = str(n).strip()\n    if not s or s.lower() == \"nan\": return None\n    base = os.path.basename(s.replace(\"\\\\\", \"/\")); stem = os.path.splitext(base)[0].lower()\n    for c in [base.lower(), stem, s.lower()] + [stem + e for e in (\".png\", \".jpg\", \".jpeg\", \".tif\", \".tiff\")]:\n        if c in IMG_FULL:  return IMG_FULL[c]\n        if c in IMG_INDEX: return IMG_INDEX[c]\n    return None\n\ndef load_messidor():\n    cands = []\n    order = ([p for p in all_csvs if \"messidor\" in p.lower()] + [p for p in all_csvs if \"messidor\" not in p.lower()])\n    for p in order:\n        if p in aptos_csv_set or \"idrid\" in p.lower(): continue\n        if \"messidor\" not in p.lower(): continue          # classify by folder name (round-12 v2 fix)\n        try: df = pd.read_csv(p)\n        except Exception: continue\n        if any(\"retinopathy grade\" in c.strip().lower() for c in df.columns): continue\n        ic = _pick(df.columns, _MI); gc = _pick(df.columns, _MG, bad=(\"dme\", \"edema\", \"macular\", \"gradab\"))\n        if ic and gc and ic != gc: cands.append((p, df, ic, gc))\n    for p, df, ic, gc in cands:\n        sub = df[[ic, gc]].dropna().copy(); sub.columns = [\"image_id\", \"grade\"]\n        try: sub[\"grade\"] = sub[\"grade\"].astype(float).round().astype(int)\n        except Exception: continue\n        sub = sub[sub[\"grade\"].between(0, 4)]\n        if not len(sub): continue\n        if not any(_resolve_any(n) for n in sub[\"image_id\"].head(20)): _unzip_matching(\"messidor\")\n        sub[\"path\"] = sub[\"image_id\"].map(_resolve_any)\n        sub = sub.dropna(subset=[\"path\"]).reset_index(drop=True)\n        if not len(sub): continue\n        sub[\"source\"] = \"Messidor-2\"\n        print(f\"Messidor-2 csv: {p} | usable {len(sub)} | grades {dict(sorted(Counter(sub['grade']).items()))}\")\n        return sub\n    print(\"Messidor-2: no usable csv found.\")\n    return None\n\nidrid_df = load_idrid()\nmessidor_df = load_messidor()\nmissing = [n for n, d in ((\"IDRiD\", idrid_df), (\"Messidor-2\", messidor_df)) if d is None]\nif missing and cfg.REQUIRE_EXTERNAL:\n    raise RuntimeError(f\"STOP: {missing} not found under {INPUT_ROOT}. Attach the dataset(s) or set cfg.REQUIRE_EXTERNAL=False.\")\n\n# ---- APTOS unlabelled test images (1,928) -> pseudo-label pool. NOT our test set. ----\nunl_df = None\nfor p in all_csvs:\n    if any(k in p.lower() for k in [\"messidor\", \"idrid\"]): continue\n    try: df = pd.read_csv(p)\n    except Exception: continue\n    cols = [c.strip().lower() for c in df.columns]\n    if \"id_code\" in cols and not any(\"diagnosis\" in c for c in cols) and len(df) > 10:\n        d = pd.DataFrame({\"image_id\": df[df.columns[cols.index(\"id_code\")]].astype(str)})\n        d[\"path\"] = d[\"image_id\"].map(_res_aptos)\n        d = d.dropna(subset=[\"path\"])\n        d = d[~d[\"image_id\"].str.lower().isin(set(aptos_df[\"image_id\"].astype(str).str.lower()))]\n        if len(d): unl_df = d.reset_index(drop=True); print(f\"APTOS unlabelled: {len(unl_df)} <- {p}\"); break\nif unl_df is None:\n    print(\"[WARN] APTOS test.csv/test_images not found -> no unlabelled pool (pseudo-labelling disabled).\")\n    unl_df = pd.DataFrame(columns=[\"image_id\", \"path\"])\nunl_df[\"grade\"] = -1; unl_df[\"source\"] = \"APTOS-unlabelled\"","metadata":{},"outputs":[],"execution_count":null},{"id":"c80fb784-31d9-4d22-834c-3705005e00b9","cell_type":"code","source":"# ---- MD5 dedup + split: byte-for-byte the round-5..10 protocol -> same 708-image test set ----\nframes = [aptos_df[[\"image_id\", \"grade\", \"path\", \"source\"]]]\nfor d in (idrid_df, messidor_df):\n    if d is not None and len(d): frames.append(d[[\"image_id\", \"grade\", \"path\", \"source\"]])\nall_df = pd.concat(frames, ignore_index=True)\nprint(\"Combined (pre-dedupe):\", len(all_df))\n\ndef _md5(p, chunk=1 << 16):\n    h = hashlib.md5()\n    try:\n        with open(p, \"rb\") as f:\n            for b in iter(lambda: f.read(chunk), b\"\"): h.update(b)\n        return h.hexdigest()\n    except Exception: return None\nif cfg.MD5_DEDUPE:\n    all_df[\"_md5\"] = [_md5(p) for p in all_df[\"path\"]]\n    dup = all_df[\"_md5\"].duplicated(keep=\"first\") & all_df[\"_md5\"].notna()\n    print(f\"MD5 dedupe: dropping {int(dup.sum())} exact duplicates {all_df.loc[dup, 'source'].value_counts().to_dict()}\")\n    all_df = all_df.loc[~dup].reset_index(drop=True).drop(columns=[\"_md5\"])\n\naptos_only = all_df[all_df[\"source\"] == \"APTOS\"].reset_index(drop=True)\ntrain_ap, temp_ap = train_test_split(aptos_only, test_size=cfg.VAL_FRAC + cfg.TEST_FRAC,\n                                     stratify=aptos_only[\"grade\"], random_state=cfg.SEED)\nval_ap, test_ap = train_test_split(temp_ap, test_size=cfg.TEST_FRAC / (cfg.VAL_FRAC + cfg.TEST_FRAC),\n                                   stratify=temp_ap[\"grade\"], random_state=cfg.SEED)\nextra = all_df[(all_df[\"source\"] != \"APTOS\") & (all_df[\"grade\"].isin(cfg.EXTRA_GRADES))]\ntrain_df = pd.concat([train_ap, extra], ignore_index=True).sample(frac=1, random_state=cfg.SEED).reset_index(drop=True)\n\nparts = []\nfor nm, d in ((\"train\", train_df), (\"val\", val_ap), (\"test\", test_ap), (\"unl\", unl_df)):\n    d = d[[\"image_id\", \"grade\", \"path\", \"source\"]].copy(); d[\"split\"] = nm; parts.append(d)\nmaster = pd.concat(parts, ignore_index=True)\nmaster[\"key\"] = master[\"source\"] + \":\" + master[\"image_id\"].astype(str)\nmaster[\"grade\"] = master[\"grade\"].astype(int)\nfor a, b in ((\"train\", \"val\"), (\"train\", \"test\"), (\"val\", \"test\")):\n    assert not (set(master.path[master.split == a]) & set(master.path[master.split == b])), f\"LEAK {a}/{b}\"\nn_test = int((master.split == \"test\").sum())\nprint(\"\\nSPLIT  \" + \" | \".join(f\"{s} {int((master.split == s).sum())}\" for s in (\"train\", \"val\", \"test\", \"unl\")))\nfor s in (\"train\", \"val\", \"test\"):\n    print(f\"  {s:5s} grades {dict(sorted(Counter(master.grade[master.split == s]).items()))}\"\n          + (f\" | sources {dict(Counter(master.source[master.split == s]))}\" if s == \"train\" else \"\"))\nif not SMOKE:\n    exp = {0: 360, 1: 68, 2: 187, 3: 37, 4: 56}\n    got = dict(sorted(Counter(master.grade[master.split == \"test\"]).items()))\n    print(\"  test matches rounds 5-10 (708 / {0:360,1:68,2:187,3:37,4:56}):\", got == exp)","metadata":{},"outputs":[],"execution_count":null},{"id":"ec4ce369-188f-42b4-a52c-cf377b1f4a52","cell_type":"code","source":"# ---- preprocessing cache: fundus crop -> pad square -> resize -> LAB-L CLAHE (RGB kept) ----\ndef preprocess(path, S=None):\n    S = S or cfg.CACHE_SIZE\n    img = cv2.imread(path, cv2.IMREAD_COLOR)\n    if img is None: raise RuntimeError(f\"unreadable image {path}\")\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    g = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY); m = g > cfg.CROP_THRESHOLD\n    r, c = np.where(m.any(1))[0], np.where(m.any(0))[0]\n    if len(r) > 10 and len(c) > 10: img = img[r[0]:r[-1] + 1, c[0]:c[-1] + 1]\n    h, w = img.shape[:2]; s = max(h, w)\n    canvas = np.zeros((s, s, 3), np.uint8); y0, x0 = (s - h) // 2, (s - w) // 2\n    canvas[y0:y0 + h, x0:x0 + w] = img\n    img = cv2.resize(canvas, (S, S), interpolation=cv2.INTER_AREA)\n    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n    lab[..., 0] = cv2.createCLAHE(clipLimit=cfg.CLAHE_CLIP, tileGridSize=(cfg.CLAHE_GRID, cfg.CLAHE_GRID)).apply(lab[..., 0])\n    return cv2.cvtColor(lab, cv2.COLOR_LAB2RGB)\n\ndef _prep_job(a): return a[0], preprocess(a[1], a[2])\n\n_sig = hashlib.md5((\"|\".join(master.path) + str(cfg.CACHE_SIZE) + str(cfg.CROP_THRESHOLD) + str(cfg.CLAHE_CLIP)).encode()).hexdigest()[:10]\nCACHE_PATH = f\"{CACHE_DIR}/cache_{cfg.CACHE_SIZE}_{_sig}.npy\"\nif not os.path.exists(CACHE_PATH):\n    t0 = time.time(); tmp = CACHE_PATH + \".part\"\n    mm = np.lib.format.open_memmap(tmp, mode=\"w+\", dtype=np.uint8, shape=(len(master), cfg.CACHE_SIZE, cfg.CACHE_SIZE, 3))\n    jobs = [(i, p, cfg.CACHE_SIZE) for i, p in enumerate(master.path)]\n    if SMOKE:\n        for j in jobs: i, a = _prep_job(j); mm[i] = a\n    else:\n        import multiprocessing as mp\n        with mp.get_context(\"fork\").Pool(max(1, os.cpu_count())) as pool:\n            for k, (i, a) in enumerate(pool.imap_unordered(_prep_job, jobs, chunksize=8)):\n                mm[i] = a\n                if k % 1000 == 0: print(f\"  cached {k}/{len(jobs)}  {time.time() - t0:.0f}s\", flush=True)\n    mm.flush(); del mm; os.replace(tmp, CACHE_PATH)\n    print(f\"cache built: {CACHE_PATH}  ({time.time() - t0:.0f}s)\")\nCACHE_ARR = np.load(CACHE_PATH, mmap_mode=\"r\")\nmaster[\"ci\"] = np.arange(len(master))\n\n# ---- near-duplicate purge (perceptual dHash, APTOS has many re-encoded copies of the same eye) ----\ndef dhash256(img):\n    g = cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2GRAY)\n    g = cv2.resize(g, (17, 16), interpolation=cv2.INTER_AREA).astype(np.int16)\n    return np.packbits((g[:, 1:] > g[:, :-1]).ravel())\nH = np.stack([dhash256(CACHE_ARR[i]) for i in range(len(master))])\ndef near_any(src_idx, ref_idx, thr):\n    \"\"\"bool per src row: is there a ref row within `thr` bits (excluding itself)?\"\"\"\n    out = np.zeros(len(src_idx), bool)\n    if len(src_idx) == 0 or len(ref_idx) == 0: return out\n    R = H[ref_idx]\n    for s in range(0, len(src_idx), 128):\n        blk = src_idx[s:s + 128]\n        d = np.unpackbits(H[blk][:, None, :] ^ R[None, :, :], axis=2).sum(2)\n        d[blk[:, None] == ref_idx[None, :]] = 999\n        out[s:s + 128] = (d <= thr).any(1)\n    return out\nidx_of = lambda s: master.index[master.split == s].values\nevalrefs = np.concatenate([idx_of(\"val\"), idx_of(\"test\")])\nothers = np.concatenate([idx_of(\"train\"), idx_of(\"val\"), idx_of(\"unl\")])\nmaster[\"test_has_dup\"] = False\nmaster.loc[idx_of(\"test\"), \"test_has_dup\"] = near_any(idx_of(\"test\"), others, cfg.DHASH_HAMMING)\nmaster[\"purged\"] = False\nif cfg.PURGE_NEAR_DUPS:\n    for s in (\"train\", \"unl\"):\n        ii = idx_of(s); hit = near_any(ii, evalrefs, cfg.DHASH_HAMMING)\n        master.loc[ii[hit], \"purged\"] = True\n        print(f\"purge: {int(hit.sum())} {s} images are near-duplicates of val/test -> removed from {s}\")\nnd = int(master.test_has_dup.sum())\nprint(f\"{nd}/{n_test} test images have a near-duplicate elsewhere in the data \"\n      f\"(reported separately as 'clean-test' accuracy on the other {n_test - nd}).\")\n\nTRAIN_IDX = master.index[(master.split == \"train\") & ~master.purged].values\nVAL_IDX, TEST_IDX = idx_of(\"val\"), idx_of(\"test\")\nUNL_IDX = master.index[(master.split == \"unl\") & ~master.purged].values\nY = master.grade.values\nprint(f\"final: train {len(TRAIN_IDX)} | val {len(VAL_IDX)} | test {len(TEST_IDX)} | unlabelled {len(UNL_IDX)}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"1089a06f-4159-483e-8ff0-7aeda15b733f","cell_type":"code","source":"# ============================ dataset ============================\ndef augment(img, size, strong, rng):\n    S = img.shape[0]\n    ang = rng.uniform(0, 360)\n    sc = rng.uniform(0.85, 1.15) if strong else rng.uniform(0.92, 1.08)\n    tx, ty = rng.uniform(-0.04, 0.04, 2) * size\n    M = cv2.getRotationMatrix2D((S / 2, S / 2), ang, sc * size / S)\n    M[0, 2] += size / 2 - S / 2 + tx; M[1, 2] += size / 2 - S / 2 + ty\n    out = cv2.warpAffine(img, M, (size, size), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    if rng.random() < 0.5: out = out[:, ::-1]\n    if rng.random() < 0.5: out = out[::-1]\n    a = rng.uniform(0.8, 1.2) if strong else rng.uniform(0.9, 1.1)\n    b = rng.uniform(-20, 20) if strong else rng.uniform(-10, 10)\n    out = out.astype(np.float32) * a + b\n    if strong:\n        if rng.random() < 0.5:   # gamma\n            out = 255.0 * (np.clip(out, 0, 255) / 255.0) ** rng.uniform(0.8, 1.25)\n        if rng.random() < 0.3:   # blur\n            out = cv2.GaussianBlur(out, (0, 0), rng.uniform(0.3, 1.2))\n    return np.clip(out, 0, 255).astype(np.uint8)\n\nclass RetinaDS(Dataset):\n    def __init__(self, idx, targets, train, size, strong=False):\n        self.idx, self.t, self.train, self.size, self.strong = np.asarray(idx), targets, train, size, strong\n        self.arr = None\n    def __len__(self): return len(self.idx)\n    def __getitem__(self, k):\n        if self.arr is None: self.arr = np.load(CACHE_PATH, mmap_mode=\"r\")\n        img = np.asarray(self.arr[self.idx[k]])\n        if self.train:\n            img = augment(img, self.size, self.strong, np.random.default_rng())\n        elif img.shape[0] != self.size:\n            img = cv2.resize(img, (self.size, self.size), interpolation=cv2.INTER_AREA)\n        x = torch.from_numpy(np.ascontiguousarray(img)).permute(2, 0, 1)\n        t = torch.from_numpy(self.t[k]) if self.t is not None else torch.zeros(cfg.NUM_CLASSES)\n        return x, t\n\ndef onehot(labels):\n    t = np.zeros((len(labels), cfg.NUM_CLASSES), np.float32); t[np.arange(len(labels)), labels] = 1; return t\n\ndef make_loader(idx, targets=None, train=False, sampler_labels=None):\n    ds = RetinaDS(idx, targets, train, cfg.IMG_SIZE, cfg.STRONG_AUG)\n    kw = dict(num_workers=cfg.NUM_WORKERS, pin_memory=device.type == \"cuda\", persistent_workers=cfg.NUM_WORKERS > 0)\n    if not train:\n        return DataLoader(ds, batch_size=cfg.BATCH_SIZE * 2, shuffle=False, **kw)\n    if cfg.SAMPLER_POWER > 0 and sampler_labels is not None:\n        cnt = np.bincount(sampler_labels, minlength=cfg.NUM_CLASSES).astype(float)\n        w = (1.0 / np.maximum(cnt, 1)) ** cfg.SAMPLER_POWER\n        smp = WeightedRandomSampler(torch.as_tensor(w[sampler_labels], dtype=torch.double), len(ds), replacement=True)\n        return DataLoader(ds, batch_size=cfg.BATCH_SIZE, sampler=smp, drop_last=True, **kw)\n    return DataLoader(ds, batch_size=cfg.BATCH_SIZE, shuffle=True, drop_last=True, **kw)\n\n# ============================ model ============================\nN_OUT = cfg.NUM_CLASSES + (1 + (cfg.NUM_CLASSES - 1) if cfg.MULTITASK else 0)\n\nclass Net(nn.Module):\n    def __init__(self):\n        super().__init__()\n        last = None\n        for name in cfg.MODEL_NAMES:\n            try:\n                self.backbone = timm.create_model(name, pretrained=cfg.PRETRAINED, num_classes=0,\n                                                  drop_path_rate=cfg.DROP_PATH, **cfg.MODEL_KW)\n                self.name = name; break\n            except Exception as e:\n                last = e; print(f\"  [model] could not load {name}: {str(e)[:120]}\")\n        else:\n            raise RuntimeError(f\"no backbone from {cfg.MODEL_NAMES} could be created: {last}\")\n        pc = getattr(self.backbone, \"pretrained_cfg\", {}) or {}\n        mean, std = pc.get(\"mean\", (0.485, 0.456, 0.406)), pc.get(\"std\", (0.229, 0.224, 0.225))\n        self.register_buffer(\"mean\", torch.tensor(mean).view(1, 3, 1, 1) * 255)\n        self.register_buffer(\"std\", torch.tensor(std).view(1, 3, 1, 1) * 255)\n        self.drop = nn.Dropout(cfg.DROP)\n        self.head = nn.Linear(self.backbone.num_features, N_OUT)\n    def forward(self, x):                       # x: uint8 or float in 0..255\n        x = (x.float() - self.mean) / self.std\n        return self.head(self.drop(self.backbone(x)))\n\ndef unwrap(m): return m.module if isinstance(m, nn.DataParallel) else m\n\nclass EMA:\n    def __init__(self, model, decay):\n        self.m = copy.deepcopy(unwrap(model)).eval(); self.decay = decay; self.n = 0\n        for p in self.m.parameters(): p.requires_grad_(False)\n    @torch.no_grad()\n    def update(self, model):\n        self.n += 1; d = min(self.decay, (1 + self.n) / (10 + self.n))\n        msd = unwrap(model).state_dict()\n        for k, v in self.m.state_dict().items():\n            if v.dtype.is_floating_point: v.mul_(d).add_(msd[k].detach(), alpha=1 - d)\n            else: v.copy_(msd[k])\n\n# ============================ loss / metrics ============================\n_GR = torch.arange(cfg.NUM_CLASSES).float()\ndef compute_loss(out, t):\n    out = out.float(); t = t.float()\n    ts = t * (1 - cfg.LABEL_SMOOTH) + cfg.LABEL_SMOOTH / cfg.NUM_CLASSES\n    loss = -(ts * F.log_softmax(out[:, :cfg.NUM_CLASSES], 1)).sum(1).mean()\n    if cfg.MULTITASK:\n        C = cfg.NUM_CLASSES\n        rt = (t * _GR.to(t.device)).sum(1)\n        loss = loss + cfg.W_REG * F.smooth_l1_loss(out[:, C], rt)\n        ot = torch.stack([t[:, k + 1:].sum(1) for k in range(C - 1)], 1)\n        loss = loss + cfg.W_ORD * F.binary_cross_entropy_with_logits(out[:, C + 1:], ot)\n    return loss\n\ndef metrics(y, pred):\n    return dict(acc=100 * accuracy_score(y, pred), qwk=100 * cohen_kappa_score(y, pred, weights=\"quadratic\"),\n                f1m=100 * f1_score(y, pred, average=\"macro\"))\n\n# ============================ inference ============================\ndef _views(x, n):\n    v = [x]\n    if n >= 2: v.append(torch.flip(x, [3]))\n    if n >= 4: v += [torch.flip(x, [2]), torch.flip(x, [2, 3])]\n    if n >= 8: v += [torch.rot90(z, 1, [2, 3]) for z in list(v)]\n    return v\n\n@torch.no_grad()\ndef predict_raw(model, idx, tta=1):\n    \"\"\"returns mean raw outputs: softmax over the class logits, sigmoid over ordinal logits, raw regression\"\"\"\n    model.eval(); outs = []\n    for x, _ in make_loader(idx):\n        x = x.to(device, non_blocking=True).float().contiguous(memory_format=MF)\n        acc = 0\n        for v in _views(x, tta):\n            with torch.autocast(\"cuda\", enabled=USE_AMP):\n                o = model(v).float()\n            C = cfg.NUM_CLASSES\n            parts = [F.softmax(o[:, :C], 1)]\n            if cfg.MULTITASK: parts += [o[:, C:C + 1], torch.sigmoid(o[:, C + 1:])]\n            acc = acc + torch.cat(parts, 1)\n        outs.append((acc / len(_views(x, tta))).cpu().numpy())\n    return np.concatenate(outs)\n\ndef to_probs(raw, mode=\"cls\", sigma=0.7):\n    \"\"\"turn raw outputs into a 5-class probability vector. modes: cls | reg | ord | cls+ord | cls*reg\"\"\"\n    C = cfg.NUM_CLASSES; p = raw[:, :C]\n    if mode == \"cls\" or not cfg.MULTITASK: return p\n    r = raw[:, C]; cum = raw[:, C + 1:]\n    preg = np.exp(-0.5 * ((np.arange(C)[None] - r[:, None]) / sigma) ** 2); preg /= preg.sum(1, keepdims=True)\n    cumf = np.concatenate([np.ones((len(cum), 1)), np.minimum.accumulate(cum, axis=1), np.zeros((len(cum), 1))], 1)\n    pord = np.clip(cumf[:, :-1] - cumf[:, 1:], 1e-6, None); pord /= pord.sum(1, keepdims=True)\n    if mode == \"reg\": return preg\n    if mode == \"ord\": return pord\n    if mode == \"cls+ord\": q = 0.5 * p + 0.5 * pord; return q / q.sum(1, keepdims=True)\n    if mode == \"cls*reg\": q = p * preg + 1e-9; return q / q.sum(1, keepdims=True)\n    raise ValueError(mode)\n\n# ============================ training ============================\ndef build_optimizer(model):\n    m = unwrap(model); groups = {\"bb_d\": [], \"bb_nd\": [], \"hd\": []}\n    for n, p in m.named_parameters():\n        if not p.requires_grad: continue\n        if n.startswith(\"head\"): groups[\"hd\"].append(p)\n        elif p.ndim <= 1 or n.endswith(\".bias\"): groups[\"bb_nd\"].append(p)\n        else: groups[\"bb_d\"].append(p)\n    return torch.optim.AdamW([\n        dict(params=groups[\"bb_d\"], lr=cfg.LR, weight_decay=cfg.WD),\n        dict(params=groups[\"bb_nd\"], lr=cfg.LR, weight_decay=0.0),\n        dict(params=groups[\"hd\"], lr=cfg.LR * cfg.HEAD_LR_MULT, weight_decay=cfg.WD)])\n\ndef val_score(model, idx):\n    raw = predict_raw(model, idx, tta=1); pred = raw[:, :cfg.NUM_CLASSES].argmax(1); m = metrics(Y[idx], pred)\n    m[\"score\"] = cfg.SELECT_W_ACC * m[\"acc\"] + (1 - cfg.SELECT_W_ACC) * m[\"qwk\"]; return m\n\ndef train_one(tag, seed, tr_idx, tr_targets, val_idx, budget_s):\n    \"\"\"trains one model; returns the best (raw or EMA) weights by val score. budget_s caps wall time.\"\"\"\n    set_seed(seed); t0 = time.time()\n    hard = tr_targets.argmax(1)\n    loader = make_loader(tr_idx, tr_targets, train=True, sampler_labels=hard)\n    model = Net().to(device).to(memory_format=MF)\n    print(f\"[{tag}] backbone {model.name} | train {len(tr_idx)} | val {len(val_idx)} | budget {budget_s / 60:.0f} min\")\n    ema = EMA(model, cfg.EMA_DECAY)\n    if cfg.MULTI_GPU and N_GPU > 1: model = nn.DataParallel(model)\n    opt = build_optimizer(model); scaler = torch.amp.GradScaler(\"cuda\", enabled=USE_AMP)\n    steps_per_epoch = max(1, len(loader) // cfg.ACCUM); plan = {\"epochs\": cfg.EPOCHS}\n    base = [g[\"lr\"] for g in opt.param_groups]\n    def set_lr(step):\n        tot = plan[\"epochs\"] * steps_per_epoch; wu = cfg.WARMUP_EPOCHS * steps_per_epoch\n        f = (step + 1) / max(1, wu) if step < wu else 0.01 + 0.99 * 0.5 * (1 + math.cos(math.pi * min(1.0, (step - wu) / max(1, tot - wu))))\n        for g, b in zip(opt.param_groups, base): g[\"lr\"] = b * f\n    best = dict(score=-1); step = 0; ep = 0; bad_steps = 0; hist = []\n    while ep < plan[\"epochs\"]:\n        te = time.time(); model.train(); tl, nb = 0.0, 0; opt.zero_grad(set_to_none=True)\n        for it, (x, t) in enumerate(loader):\n            x = x.to(device, non_blocking=True).float().contiguous(memory_format=MF)\n            t = t.to(device, non_blocking=True).float()\n            if cfg.MIXUP_P > 0 and np.random.rand() < cfg.MIXUP_P:\n                lam = float(np.random.beta(cfg.MIXUP_ALPHA, cfg.MIXUP_ALPHA)); perm = torch.randperm(x.size(0), device=x.device)\n                x = lam * x + (1 - lam) * x[perm]; t = lam * t + (1 - lam) * t[perm]\n            with torch.autocast(\"cuda\", enabled=USE_AMP):\n                out = model(x)\n            loss = compute_loss(out, t)\n            if not torch.isfinite(loss):\n                bad_steps += 1; opt.zero_grad(set_to_none=True); continue\n            scaler.scale(loss / cfg.ACCUM).backward()\n            if (it + 1) % cfg.ACCUM == 0:\n                set_lr(step); scaler.unscale_(opt)\n                nn.utils.clip_grad_norm_(model.parameters(), cfg.GRAD_CLIP)\n                scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True)\n                ema.update(model); step += 1\n            tl += float(loss); nb += 1\n        mr = val_score(model, val_idx); me = val_score(ema.m, val_idx)\n        for kind, m, src in ((\"raw\", mr, unwrap(model)), (\"ema\", me, ema.m)):\n            if m[\"score\"] > best[\"score\"]:\n                best = dict(score=m[\"score\"], acc=m[\"acc\"], qwk=m[\"qwk\"], epoch=ep + 1, kind=kind,\n                            state={k: v.detach().cpu().clone() for k, v in src.state_dict().items()})\n        dt = time.time() - te\n        hist.append(dict(epoch=ep + 1, loss=tl / max(1, nb), raw_acc=mr[\"acc\"], raw_qwk=mr[\"qwk\"], ema_acc=me[\"acc\"], ema_qwk=me[\"qwk\"], sec=dt))\n        print(f\"  [{tag}] ep {ep + 1:2d}/{plan['epochs']} loss {tl / max(1, nb):.4f} | raw acc {mr['acc']:.2f} qwk {mr['qwk']:.2f}\"\n              f\" | ema acc {me['acc']:.2f} qwk {me['qwk']:.2f} | best {best['acc']:.2f} ({best['kind']}@{best['epoch']})\"\n              f\" | {dt:.0f}s\" + (f\" | {bad_steps} non-finite steps skipped\" if bad_steps else \"\"), flush=True)\n        ep += 1\n        if ep == 1:   # plan the number of epochs that fits in the budget (the cosine schedule follows the plan)\n            fit = int((budget_s - (time.time() - t0)) / (dt * 1.05)) + 1\n            if fit < plan[\"epochs\"]:\n                plan[\"epochs\"] = max(2, fit); print(f\"  [{tag}] budget -> training {plan['epochs']} epochs instead of {cfg.EPOCHS}\")\n        elif time.time() - t0 + dt * 1.05 > budget_s and ep < plan[\"epochs\"]:\n            print(f\"  [{tag}] stopping early at epoch {ep}: time budget reached\"); break\n    net = copy.deepcopy(ema.m); net.load_state_dict(best[\"state\"])\n    net = net.to(device).to(memory_format=MF)\n    if cfg.MULTI_GPU and N_GPU > 1: net = nn.DataParallel(net)\n    del model, ema, opt, loader; gc.collect(); torch.cuda.empty_cache()\n    print(f\"[{tag}] done in {(time.time() - t0) / 60:.1f} min | best val acc {best['acc']:.2f} qwk {best['qwk']:.2f} ({best['kind']} @ ep {best['epoch']})\")\n    return net, best, pd.DataFrame(hist)\n\ndef train_runs(tag, tr_idx, tr_targets, val_idx, n_runs, budget_s, pred_sets):\n    \"\"\"trains n_runs seeds (budget split evenly), predicts every index set in pred_sets with TTA, averages runs.\"\"\"\n    t0 = time.time(); acc = {k: [] for k in pred_sets}; bests = []\n    for r in range(n_runs):\n        left = budget_s - (time.time() - t0); share = left / (n_runs - r)\n        if r > 0 and share < 0.6 * bests[0][\"minutes\"] * 60:\n            print(f\"[{tag}] skipping run {r}: not enough time left ({left / 60:.0f} min)\"); break\n        tr0 = time.time()\n        net, best, hist = train_one(f\"{tag}-r{r}\", 1000 + 17 * r + cfg.SEED, tr_idx, tr_targets, val_idx, share)\n        for k, ii in pred_sets.items():\n            if len(ii): acc[k].append(predict_raw(net, ii, cfg.TTA))\n        if r == 0 and cfg.SAVE_WEIGHTS: torch.save(unwrap(net).state_dict(), f\"{WORK}/{RUN_NAME}_{tag}_r0.pt\")\n        best.pop(\"state\"); best[\"minutes\"] = (time.time() - tr0) / 60; bests.append(best)\n        if \"val\" in pred_sets:\n            print(f\"[{tag}] run {r} val acc with TTA: {metrics(Y[pred_sets['val']], acc['val'][-1][:, :cfg.NUM_CLASSES].argmax(1))['acc']:.2f}\")\n        del net; gc.collect(); torch.cuda.empty_cache()\n    return {k: (np.mean(v, 0) if v else None) for k, v in acc.items()}, bests","metadata":{},"outputs":[],"execution_count":null},{"id":"e37ef750-4239-4506-9ea8-4e1c86b0eedc","cell_type":"code","source":"# ---- GPU preflight: 2 real training steps in a SEPARATE process with CUDA_LAUNCH_BLOCKING=1.\n# If CUDA crashes, its exact error message is printed here instead of the kernel silently dying.\nimport subprocess, textwrap\nprint(\"code version: r13-v2 (single GPU) | MULTI_GPU =\", cfg.MULTI_GPU, \"| CHANNELS_LAST =\", cfg.CHANNELS_LAST)\nif device.type == \"cuda\":\n    pf = {\"names\": cfg.MODEL_NAMES, \"kw\": cfg.MODEL_KW, \"dp\": cfg.DROP_PATH, \"img\": cfg.IMG_SIZE, \"bs\": cfg.BATCH_SIZE,\n          \"multi\": bool(cfg.MULTI_GPU and N_GPU > 1), \"cl\": bool(cfg.CHANNELS_LAST), \"nout\": int(N_OUT)}\n    script = textwrap.dedent(\"\"\"\n        import json, sys, torch, timm\n        pf = json.loads(sys.argv[1]); m = None\n        for n in pf[\"names\"]:\n            try:\n                m = timm.create_model(n, pretrained=False, num_classes=pf[\"nout\"], drop_path_rate=pf[\"dp\"], **pf[\"kw\"]); break\n            except Exception as e: print(\"skip\", n, e)\n        mf = torch.channels_last if pf[\"cl\"] else torch.contiguous_format\n        m = m.cuda().to(memory_format=mf)\n        if pf[\"multi\"]: m = torch.nn.DataParallel(m)\n        opt = torch.optim.AdamW(m.parameters(), 1e-4); sc = torch.amp.GradScaler(\"cuda\")\n        x = torch.randn(pf[\"bs\"], 3, pf[\"img\"], pf[\"img\"], device=\"cuda\").contiguous(memory_format=mf)\n        for _ in range(2):\n            with torch.autocast(\"cuda\"):\n                loss = m(x).float().mean()\n            sc.scale(loss).backward(); sc.step(opt); sc.update(); opt.zero_grad()\n        torch.cuda.synchronize()\n        print(\"PREFLIGHT OK | batch %d @ %dpx | peak GPU memory %.1f GB\" % (pf[\"bs\"], pf[\"img\"], torch.cuda.max_memory_allocated() / 1e9))\n    \"\"\")\n    r = subprocess.run([sys.executable, \"-c\", script, json.dumps(pf)], capture_output=True, text=True,\n                       env=dict(os.environ, CUDA_LAUNCH_BLOCKING=\"1\"), timeout=1200)\n    print(r.stdout[-3000:])\n    if r.returncode != 0:\n        print(r.stderr[-8000:])\n        raise RuntimeError(f\"GPU preflight failed (exit code {r.returncode}). The text above is the real CUDA error.\")","metadata":{},"outputs":[],"execution_count":null},{"id":"256e8f0a-35b9-4cd0-b936-886a48a03027","cell_type":"code","source":"# ============================ accuracy decoder / reporting / saving ============================\ndef bias_decoder(p, y, grid=np.arange(-1.5, 1.51, 0.05), passes=3):\n    \"\"\"per-class log-bias maximising accuracy (QWK tie-break) by coordinate ascent\"\"\"\n    lp = np.log(np.clip(p, 1e-9, None)); b = np.zeros(p.shape[1])\n    def sc(bb):\n        pr = (lp + bb).argmax(1); return accuracy_score(y, pr) + 1e-3 * cohen_kappa_score(y, pr, weights=\"quadratic\")\n    cur = sc(b)\n    for _ in range(passes):\n        for c in range(1, p.shape[1]):          # class 0 is the reference\n            for v in grid:\n                bb = b.copy(); bb[c] = v; s = sc(bb)\n                if s > cur + 1e-12: cur, b = s, bb\n    return b\n\ndef decode(p, bias): return (np.log(np.clip(p, 1e-9, None)) + bias).argmax(1)\n\ndef choose_decoder(p_val, y_val):\n    b = bias_decoder(p_val, y_val)\n    a0 = 100 * accuracy_score(y_val, p_val.argmax(1)); a1 = 100 * accuracy_score(y_val, decode(p_val, b))\n    use = a1 - a0 >= cfg.DECODER_GUARD\n    print(f\"decoder: val argmax {a0:.2f} | val bias {a1:.2f} (bias {np.round(b, 2).tolist()}) -> \"\n          + (\"ADOPT bias\" if use else f\"keep argmax (gain < {cfg.DECODER_GUARD} pts guard)\"))\n    return b if use else np.zeros_like(b)\n\ndef bootstrap_ci(y, pred, n=2000, seed=0):\n    r = np.random.default_rng(seed); c = (y == pred).astype(float)\n    s = [c[r.integers(0, len(c), len(c))].mean() for _ in range(n)]\n    return 100 * np.percentile(s, 2.5), 100 * np.percentile(s, 97.5)\n\ndef report(name, y, pred, clean_mask=None):\n    m = metrics(y, pred); lo, hi = bootstrap_ci(y, pred)\n    print(\"=\" * 78); print(f\"{name}\")\n    print(f\"  ACCURACY {m['acc']:.2f}%  (95% CI {lo:.1f}-{hi:.1f})  | QWK {m['qwk']:.2f} | macro-F1 {m['f1m']:.2f} | errors {int((y != pred).sum())}/{len(y)}\")\n    if clean_mask is not None and clean_mask.sum() < len(y):\n        print(f\"  clean-test accuracy (no near-duplicate anywhere): {100 * accuracy_score(y[clean_mask], pred[clean_mask]):.2f}% on {int(clean_mask.sum())}\")\n    cm = confusion_matrix(y, pred, labels=range(cfg.NUM_CLASSES))\n    rec = cm.diagonal() / np.maximum(cm.sum(1), 1)\n    print(\"  per-class recall: \" + \" | \".join(f\"{n} {100 * r:.1f}\" for n, r in zip(cfg.CLASS_NAMES, rec)))\n    print(\"  confusion matrix (rows = truth):\"); print(pd.DataFrame(cm, index=cfg.CLASS_NAMES, columns=cfg.CLASS_NAMES).to_string())\n    gap = m[\"acc\"] - cfg.TARGET\n    print(f\"  >>> {'ABOVE' if gap > 0 else 'BELOW'} TARGET {cfg.TARGET}% by {abs(gap):.2f} pts\"); print(\"=\" * 78)\n    return m\n\ndef save_member(name, p_val, p_test, p_unl, bias, info):\n    d = os.path.join(OUT_DIR, name); os.makedirs(d, exist_ok=True)\n    np.save(f\"{d}/val_probs.npy\", p_val.astype(np.float32)); np.save(f\"{d}/test_probs.npy\", p_test.astype(np.float32))\n    if p_unl is not None and len(UNL_IDX): np.save(f\"{d}/unl_probs.npy\", p_unl.astype(np.float32))\n    k = pd.concat([master.loc[VAL_IDX, [\"key\", \"grade\", \"split\"]], master.loc[TEST_IDX, [\"key\", \"grade\", \"split\", \"test_has_dup\"]],\n                   master.loc[UNL_IDX, [\"key\", \"grade\", \"split\"]] if p_unl is not None else None])\n    k.to_csv(f\"{d}/keys.csv\", index=False)\n    info = dict(info, run_name=name, bias=[float(v) for v in bias], n_val=len(VAL_IDX), n_test=len(TEST_IDX),\n                n_unl=int(len(UNL_IDX)) if p_unl is not None else 0, notebook=RUN_NAME)\n    json.dump(info, open(f\"{d}/meta.json\", \"w\"), indent=2, default=float)\n    print(\"saved member ->\", d)\n\ndef finish_member(name, P, bests=None, extra_info=None, bias=None):\n    pv, pt, pu = P[\"val\"], P[\"test\"], P.get(\"unl\")\n    yv, yt = Y[VAL_IDX], Y[TEST_IDX]\n    if bias is None: bias = choose_decoder(pv, yv)\n    mv = metrics(yv, decode(pv, bias))\n    print(f\"[{name}] VAL (selection set) acc {mv['acc']:.2f} qwk {mv['qwk']:.2f}\")\n    mt = report(f\"TEST — {name}  (n={len(yt)}, 5-class, TTA x{cfg.TTA}, decoder={'bias' if np.any(bias) else 'argmax'})\",\n                yt, decode(pt, bias), ~master.loc[TEST_IDX, \"test_has_dup\"].values)\n    info = dict(val_acc=mv[\"acc\"], val_qwk=mv[\"qwk\"], test_acc=mt[\"acc\"], test_qwk=mt[\"qwk\"],\n                model=cfg.MODEL_NAMES[0], img=cfg.IMG_SIZE, runs=bests or [], **(extra_info or {}))\n    save_member(name, pv, pt, pu, bias, info)\n    return mt","metadata":{},"outputs":[],"execution_count":null},{"id":"3875221b-ec72-4efb-a453-9615f825952b","cell_type":"code","source":"# ============================ train N_RUNS seeds, predict val / test / unlabelled with TTA ============================\nbudget = hours_left() * 3600 - cfg.RESERVE_MIN * 60\nR, bests = train_runs(\"main\", TRAIN_IDX, onehot(Y[TRAIN_IDX]), VAL_IDX, cfg.N_RUNS, budget, {\"val\": VAL_IDX, \"test\": TEST_IDX, \"unl\": UNL_IDX if cfg.PREDICT_UNL else np.array([], int)})\nP = {k: (to_probs(v, \"cls\") if v is not None else None) for k, v in R.items()}","metadata":{},"outputs":[],"execution_count":null},{"id":"60d9fa51-319a-4cd8-9bc0-a13c3b52601e","cell_type":"code","source":"# ============================ decoder (chosen on val) + the ONE test report + save for notebooks 08/10 ============================\nfinal = finish_member(RUN_NAME, P, bests)\nprint(f\"total runtime {(time.time() - T_START) / 3600:.2f} h\")","metadata":{},"outputs":[],"execution_count":null}]}