{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# APTOS 2019 Blindness Detection: Pretrained CNN (EfficientNet-B0)\n**Course:** Artificial Neural Networks | **Category:** Pretrained CNN | **Metric:** Quadratic Weighted Kappa (QWK, higher is better)\n\n**Task:** grade diabetic retinopathy (0 = No DR, 1 = Mild, 2 = Moderate, 3 = Severe, 4 = Proliferative) from a retina photograph.\n\n**Models compared in this notebook**\n| ID | Description |\n|---|---|\n| M0 (baseline) | EfficientNet-B0, frozen backbone, 224x224 plain resize, 5-class softmax, no augmentation |\n| M1 (improved) | EfficientNet-B0 fine-tuned, 256x256, crop + Ben Graham contrast enhancement, augmentation, regression head, LR scheduling |\n| M2 (improved) | M1 + flip test-time augmentation (TTA) + QWK-optimised thresholds |\n\n**How to submit each model:** change `SUBMIT_AS` in the setup cell to `\"M0\"`, `\"M1\"` or `\"M2\"`, then Save Version and submit.","metadata":{}},{"cell_type":"markdown","source":"## 0. Setup: library versions, seeds, configuration","metadata":{}},{"cell_type":"code","source":"import os, glob, time, random, warnings\nimport numpy as np, pandas as pd\nimport cv2, matplotlib.pyplot as plt\nimport tensorflow as tf, sklearn\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom concurrent.futures import ThreadPoolExecutor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, confusion_matrix, classification_report\nwarnings.filterwarnings(\"ignore\")\n\n# ---- reproducibility: fix every random seed ----\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED); tf.random.set_seed(SEED)\nkeras.utils.set_random_seed(SEED)\n\n# ---- library versions ----\nprint(\"tensorflow  :\", tf.__version__)\nprint(\"keras       :\", keras.__version__)\nprint(\"numpy       :\", np.__version__)\nprint(\"pandas      :\", pd.__version__)\nprint(\"scikit-learn:\", sklearn.__version__)\nprint(\"opencv      :\", cv2.__version__)\nprint(\"GPUs        :\", tf.config.list_physical_devices(\"GPU\"))\n\n# ---- configuration ----\nSUBMIT_AS = \"M4\"    # \"M0\", \"M1\" or \"M2\": which model writes submission.csv\nDEBUG = False       # True = tiny quick run to test the code, False = real run","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:22:15.314363Z","iopub.execute_input":"2026-10-05T11:22:15.314615Z","iopub.status.idle":"2026-10-05T11:22:39.516958Z","shell.execute_reply.started":"2026-10-05T11:22:15.314584Z","shell.execute_reply":"2026-10-05T11:22:39.51603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> 📸 **Screenshot 1 (Student note, delete before submission):** output of this cell (versions + GPU line).","metadata":{}},{"cell_type":"markdown","source":"## 1. Data location and overview","metadata":{}},{"cell_type":"code","source":"# Find the competition data folder automatically (Kaggle may use different base paths)\ntrain_csv = [p for p in glob.glob(\"/kaggle/input/**/train.csv\", recursive=True) if \"aptos\" in p.lower()]\nassert train_csv, \"Competition data not attached: use Add Input and add APTOS 2019 Blindness Detection\"\nDATA = os.path.dirname(train_csv[0])\nTRAIN_DIR, TEST_DIR = f\"{DATA}/train_images\", f\"{DATA}/test_images\"\nprint(\"Data path:\", DATA)\nprint(\"Files:\", os.listdir(DATA))\n\ntrain  = pd.read_csv(f\"{DATA}/train.csv\")\ntest   = pd.read_csv(f\"{DATA}/test.csv\")\nsample = pd.read_csv(f\"{DATA}/sample_submission.csv\")\nprint(\"train:\", train.shape, \"| test:\", test.shape, \"| sample_submission:\", sample.shape)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:23:29.22196Z","iopub.execute_input":"2026-10-05T11:23:29.222306Z","iopub.status.idle":"2026-10-05T11:23:38.249832Z","shell.execute_reply.started":"2026-10-05T11:23:29.22228Z","shell.execute_reply":"2026-10-05T11:23:38.249124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CLASS_NAMES = [\"0 No DR\", \"1 Mild\", \"2 Moderate\", \"3 Severe\", \"4 Proliferative\"]\nprint(\"Input  : retina photograph (PNG, different sizes)\")\nprint(\"Target : diagnosis (integer 0-4)  | classes:\", train.diagnosis.nunique())\nprint(\"Train images:\", len(train), \"| Test images:\", len(test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:24:01.72816Z","iopub.execute_input":"2026-10-05T11:24:01.728513Z","iopub.status.idle":"2026-10-05T11:24:01.737378Z","shell.execute_reply.started":"2026-10-05T11:24:01.728488Z","shell.execute_reply":"2026-10-05T11:24:01.736589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Exploratory Data Analysis (EDA)\n### 2.1 Target distribution","metadata":{}},{"cell_type":"code","source":"counts = train.diagnosis.value_counts().sort_index()\npct = (counts / len(train) * 100).round(1)\nprint(pd.DataFrame({\"class\": CLASS_NAMES, \"count\": counts.values, \"percent\": pct.values}))\n\nplt.figure(figsize=(7, 4))\nbars = plt.bar(CLASS_NAMES, counts.values, color=\"steelblue\")\nfor b, v in zip(bars, counts.values):\n    plt.text(b.get_x() + b.get_width()/2, v + 10, str(v), ha=\"center\")\nplt.title(\"Training set: class distribution\"); plt.ylabel(\"images\"); plt.xticks(rotation=20)\nplt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:24:05.401798Z","iopub.execute_input":"2026-10-05T11:24:05.402138Z","iopub.status.idle":"2026-10-05T11:24:05.665913Z","shell.execute_reply.started":"2026-10-05T11:24:05.402112Z","shell.execute_reply":"2026-10-05T11:24:05.664942Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Observation (write after seeing the plot):** class 0 is the largest class (about half the data) while classes 3 and 4 are small, so the data is **imbalanced**. Accuracy alone would be misleading, which is why QWK is used.\n\n> 📸 **Screenshot 2 (delete this note before submission):** this class-distribution plot.","metadata":{}},{"cell_type":"markdown","source":"### 2.2 Data quality checks","metadata":{}},{"cell_type":"code","source":"print(\"Missing values per column:\"); print(train.isna().sum())\nprint(\"\\nDuplicate id_code values:\", train.id_code.duplicated().sum())\nmissing_files = [i for i in train.id_code[:200] if not os.path.exists(f\"{TRAIN_DIR}/{i}.png\")]\nprint(\"Missing image files (first 200 checked):\", len(missing_files))\n\n# image sizes on a random sample of 100 images (raw images are large)\nrng = np.random.default_rng(SEED)\nsizes = []\nfor i in rng.choice(train.id_code.values, 100, replace=False):\n    img = cv2.imread(f\"{TRAIN_DIR}/{i}.png\")\n    sizes.append(img.shape[:2])\nsizes = pd.Series(sizes).value_counts()\nprint(\"\\nDistinct image sizes in a 100-image sample:\", len(sizes))\nprint(sizes.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:24:11.545236Z","iopub.execute_input":"2026-10-05T11:24:11.545602Z","iopub.status.idle":"2026-10-05T11:24:23.150288Z","shell.execute_reply.started":"2026-10-05T11:24:11.545571Z","shell.execute_reply":"2026-10-05T11:24:23.149488Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Observation (write after seeing output):** state how many missing values, duplicates, and how many different image sizes were found. Different sizes mean every image must be resized to one fixed size.","metadata":{}},{"cell_type":"markdown","source":"### 2.3 Sample images","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(5, 3, figsize=(10, 14))\nfor cls in range(5):\n    ids = train[train.diagnosis == cls].id_code.sample(3, random_state=SEED).values\n    for j, i in enumerate(ids):\n        img = cv2.cvtColor(cv2.imread(f\"{TRAIN_DIR}/{i}.png\"), cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (300, 300))\n        axes[cls, j].imshow(img); axes[cls, j].axis(\"off\")\n        axes[cls, j].set_title(CLASS_NAMES[cls], fontsize=9)\nplt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:25:32.0537Z","iopub.execute_input":"2026-10-05T11:25:32.054056Z","iopub.status.idle":"2026-10-05T11:25:35.447737Z","shell.execute_reply.started":"2026-10-05T11:25:32.054033Z","shell.execute_reply":"2026-10-05T11:25:35.446653Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Observation (write after seeing the images):** images differ in brightness, colour and amount of black border, and the visual difference between neighbouring grades (for example 1 vs 2) is subtle.\n\n> 📸 **Screenshot 3 (delete this note before submission):** this grid of sample images.","metadata":{}},{"cell_type":"markdown","source":"### 2.4 EDA summary (used to choose preprocessing)\n1. **Imbalance** -> use a stratified split and judge models with QWK and a confusion matrix, not accuracy alone.\n2. **Different sizes and black borders** -> crop the black border and resize to a fixed size (224 for M0, 256 for M1).\n3. **Brightness / colour variation** -> Ben Graham contrast enhancement (subtract a blurred copy) in M1 to highlight lesions.\n4. **Retina images have no natural orientation** -> flips and arbitrary rotations are safe augmentations.\n5. **Grades are ordered** -> treat the task as ordinal (regression head with rounding) in M1.","metadata":{}},{"cell_type":"markdown","source":"## 3. Preprocessing","metadata":{}},{"cell_type":"code","source":"def read_rgb(path):\n    img = cv2.imread(path)\n    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\ndef crop_black(img, tol=7):\n    \"\"\"Remove the black border around the retina.\"\"\"\n    mask = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) > tol\n    if mask.sum() == 0:\n        return img\n    rows, cols = np.where(mask.any(1))[0], np.where(mask.any(0))[0]\n    return img[rows[0]:rows[-1] + 1, cols[0]:cols[-1] + 1]\n\ndef prep_simple(img, size=224):\n    \"\"\"M0: plain resize only.\"\"\"\n    return cv2.resize(img, (size, size))\n\ndef prep_ben(img, size=256):\n    \"\"\"M1/M2: crop black border, resize, Ben Graham contrast enhancement.\"\"\"\n    img = cv2.resize(crop_black(img), (size, size))\n    blur = cv2.GaussianBlur(img, (0, 0), size / 30)\n    return cv2.addWeighted(img, 4, blur, -4, 128)\n\n# before / after on one image\nsample_id = train.id_code.iloc[0]\nraw = read_rgb(f\"{TRAIN_DIR}/{sample_id}.png\")\nfig, ax = plt.subplots(1, 3, figsize=(12, 4))\nax[0].imshow(raw);                ax[0].set_title(\"Original\")\nax[1].imshow(prep_simple(raw));   ax[1].set_title(\"M0: resize 224\")\nax[2].imshow(prep_ben(raw));      ax[2].set_title(\"M1: crop + Ben Graham 256\")\nfor a in ax: a.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:25:53.30006Z","iopub.execute_input":"2026-10-05T11:25:53.300416Z","iopub.status.idle":"2026-10-05T11:25:54.227038Z","shell.execute_reply.started":"2026-10-05T11:25:53.300392Z","shell.execute_reply":"2026-10-05T11:25:54.226131Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Why these steps:** resize gives a fixed input size for the CNN (and keeps training within the time limit); cropping removes useless black pixels; the Ben Graham filter is a well-known idea from the 2015 Kaggle Diabetic Retinopathy competition (B. Graham, 2015) that removes lighting differences between cameras.\n\n> 📸 **Screenshot 4 (delete this note before submission):** this before/after image.","metadata":{}},{"cell_type":"markdown","source":"### 3.1 Load all images into memory (done once)\nReading large PNG files every epoch would be slow, so every image is converted once into two small arrays (224 for M0, 256 for M1).","metadata":{}},{"cell_type":"code","source":"def load_one(args):\n    folder, i = args\n    img = read_rgb(f\"{folder}/{i}.png\")\n    return prep_simple(img), prep_ben(img)\n\ndef load_images(ids, folder):\n    with ThreadPoolExecutor(max_workers=8) as ex:\n        out = list(ex.map(load_one, [(folder, i) for i in ids]))\n    return np.stack([o[0] for o in out]), np.stack([o[1] for o in out])\n\nif DEBUG:\n    train = train.sample(300, random_state=SEED).reset_index(drop=True)\n\nt0 = time.time()\nX0_all, X1_all = load_images(train.id_code.values, TRAIN_DIR)\ny_all = train.diagnosis.values\nprint(X0_all.shape, X1_all.shape, y_all.shape, f\"| {time.time() - t0:.0f} s\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:26:04.808213Z","iopub.execute_input":"2026-10-05T11:26:04.808579Z","iopub.status.idle":"2026-10-05T11:26:18.525687Z","shell.execute_reply.started":"2026-10-05T11:26:04.808546Z","shell.execute_reply":"2026-10-05T11:26:18.524791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3.2 Train / validation split","metadata":{}},{"cell_type":"code","source":"idx_tr, idx_va = train_test_split(np.arange(len(train)), test_size=0.2, stratify=y_all, random_state=SEED)\n\nX0_tr, X0_va = X0_all[idx_tr], X0_all[idx_va]\nX1_tr, X1_va = X1_all[idx_tr], X1_all[idx_va]\ny_tr,  y_va  = y_all[idx_tr],  y_all[idx_va]\n\nprint(\"Split ratio: 80 / 20 | stratified: Yes | seed:\", SEED)\nprint(\"Train:\", len(idx_tr), \"| Validation:\", len(idx_va))\nprint(pd.DataFrame({\"train\": pd.Series(y_tr).value_counts().sort_index(),\n                    \"val\":   pd.Series(y_va).value_counts().sort_index()}))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:26:26.330201Z","iopub.execute_input":"2026-10-05T11:26:26.330576Z","iopub.status.idle":"2026-10-05T11:26:26.3781Z","shell.execute_reply.started":"2026-10-05T11:26:26.330544Z","shell.execute_reply":"2026-10-05T11:26:26.377338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Augmentation note:** augmentation (flip, rotation, zoom) is built into the M1 model as Keras layers. These layers are active only during training and are switched off automatically for validation and test data. M0 uses no augmentation.","metadata":{}},{"cell_type":"markdown","source":"## 4. Metric helpers","metadata":{}},{"cell_type":"code","source":"def qwk(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n\nDEFAULT_THR = [0.5, 1.5, 2.5, 3.5]\ndef to_grade(pred, thr=DEFAULT_THR):\n    \"\"\"Regression output -> integer grade 0-4 using thresholds.\"\"\"\n    return np.digitize(np.ravel(pred), thr)\n\ndef optimise_thresholds(pred, y, rounds=3):\n    \"\"\"Coordinate search: move each threshold to maximise QWK.\"\"\"\n    thr = list(DEFAULT_THR)\n    for _ in range(rounds):\n        for k in range(4):\n            best_s, best_v = qwk(y, to_grade(pred, thr)), thr[k]\n            for v in np.arange(thr[k] - 0.5, thr[k] + 0.5, 0.05):\n                t = thr.copy(); t[k] = v\n                if t != sorted(t):\n                    continue\n                s = qwk(y, to_grade(pred, t))\n                if s > best_s:\n                    best_s, best_v = s, v\n            thr[k] = best_v\n    return thr\n\nclass QWKHistory(keras.callbacks.Callback):\n    \"\"\"Records validation QWK after every epoch (for learning curves).\"\"\"\n    def __init__(self, Xv, yv, regression):\n        super().__init__(); self.Xv, self.yv, self.reg = Xv, yv, regression; self.values = []\n    def on_epoch_end(self, epoch, logs=None):\n        p = self.model.predict(self.Xv, batch_size=64, verbose=0)\n        g = to_grade(p) if self.reg else p.argmax(1)\n        self.values.append(qwk(self.yv, g))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:27:07.393329Z","iopub.execute_input":"2026-10-05T11:27:07.393713Z","iopub.status.idle":"2026-10-05T11:27:07.403713Z","shell.execute_reply.started":"2026-10-05T11:27:07.393686Z","shell.execute_reply":"2026-10-05T11:27:07.402714Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Model architectures\n### 5.0 Pretrained weights (offline)\nInternet is OFF in this competition, so the ImageNet weights of EfficientNet-B0 are attached from the helper notebook (`efficientnetb0_notop.weights.h5`). Source: Keras Applications EfficientNetB0 (Tan & Le, 2019).","metadata":{}},{"cell_type":"code","source":"def make_backbone(size):\n    base = keras.applications.EfficientNetB0(include_top=False, weights=None, input_shape=(size, size, 3))\n    found = glob.glob(\"/kaggle/input/**/efficientnetb0_notop.weights.h5\", recursive=True)\n    if found:\n        base.load_weights(found[0])\n        print(\"Loaded pretrained weights from:\", found[0])\n    else:\n        print(\"Weights file not attached, trying to download (works only with Internet ON)...\")\n        base = keras.applications.EfficientNetB0(include_top=False, weights=\"imagenet\", input_shape=(size, size, 3))\n    return base","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:27:20.690355Z","iopub.execute_input":"2026-10-05T11:27:20.690818Z","iopub.status.idle":"2026-10-05T11:27:20.696638Z","shell.execute_reply.started":"2026-10-05T11:27:20.690771Z","shell.execute_reply":"2026-10-05T11:27:20.695703Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5.1 Baseline M0: frozen EfficientNet-B0 + softmax head (224x224, no augmentation)","metadata":{}},{"cell_type":"code","source":"def build_m0():\n    base = make_backbone(224)\n    base.trainable = False                      # use as a fixed feature extractor\n    inp = keras.Input((224, 224, 3))\n    x = base(inp, training=False)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.3)(x)\n    out = layers.Dense(5, activation=\"softmax\")(x)   # 5 classes\n    return keras.Model(inp, out, name=\"M0_baseline\")\n\nm0 = build_m0()\nm0.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:27:30.377324Z","iopub.execute_input":"2026-10-05T11:27:30.378078Z","iopub.status.idle":"2026-10-05T11:27:34.176602Z","shell.execute_reply.started":"2026-10-05T11:27:30.378044Z","shell.execute_reply":"2026-10-05T11:27:34.175862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5.2 Improved M1: fine-tuned EfficientNet-B0 + augmentation + regression head (256x256)\nChanges from M0: (1) all layers fine-tuned (BatchNorm kept frozen for stability), (2) augmentation layers, (3) larger input with Ben Graham preprocessing, (4) single linear output (ordinal regression) trained with Huber loss, (5) learning-rate scheduling and more dropout.","metadata":{}},{"cell_type":"code","source":"def build_m1():\n    base = make_backbone(256)\n    base.trainable = True\n    for l in base.layers:\n        if isinstance(l, layers.BatchNormalization):\n            l.trainable = False\n    inp = keras.Input((256, 256, 3))\n    x = layers.RandomFlip(\"horizontal_and_vertical\")(inp)       # augmentation (training only)\n    x = layers.RandomRotation(0.5, fill_mode=\"constant\")(x)\n    x = layers.RandomZoom(0.1)(x)\n    x = base(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.4)(x)\n    out = layers.Dense(1)(x)                                     # one number = severity score\n    return keras.Model(inp, out, name=\"M1_improved\")\n\nm1 = build_m1()\nm1.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:27:40.705155Z","iopub.execute_input":"2026-10-05T11:27:40.705643Z","iopub.status.idle":"2026-10-05T11:27:44.545584Z","shell.execute_reply.started":"2026-10-05T11:27:40.705598Z","shell.execute_reply":"2026-10-05T11:27:44.544598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def trainable_params(model):\n    return int(sum(np.prod(v.shape) for v in model.trainable_weights))\nprint(\"M0 trainable parameters:\", f\"{trainable_params(m0):,}\")\nprint(\"M1 trainable parameters:\", f\"{trainable_params(m1):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:27:50.169864Z","iopub.execute_input":"2026-10-05T11:27:50.170379Z","iopub.status.idle":"2026-10-05T11:27:50.183317Z","shell.execute_reply.started":"2026-10-05T11:27:50.170344Z","shell.execute_reply":"2026-10-05T11:27:50.181845Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> 📸 **Screenshot 5 (delete before submission):** both `model.summary()` outputs (scroll to show the Total/Trainable params lines).","metadata":{}},{"cell_type":"markdown","source":"## 6. Training","metadata":{}},{"cell_type":"code","source":"def fit_model(model, name, Xtr, ytr, Xva, yva, lr, loss, metrics, epochs, regression):\n    model.compile(optimizer=keras.optimizers.Adam(lr), loss=loss, metrics=metrics)\n    qwk_cb = QWKHistory(Xva, yva, regression)\n    cbs = [qwk_cb,\n           keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, restore_best_weights=True),\n           keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=2, min_lr=1e-6)]\n    t0 = time.time()\n    hist = model.fit(Xtr, ytr, validation_data=(Xva, yva), epochs=epochs,\n                     batch_size=32, callbacks=cbs, verbose=2)\n    minutes = (time.time() - t0) / 60\n    print(f\"{name}: {len(hist.history['loss'])} epochs run, {minutes:.1f} min\")\n    return hist.history, qwk_cb.values, minutes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:28:16.728554Z","iopub.execute_input":"2026-10-05T11:28:16.728922Z","iopub.status.idle":"2026-10-05T11:28:16.735938Z","shell.execute_reply.started":"2026-10-05T11:28:16.728895Z","shell.execute_reply":"2026-10-05T11:28:16.735089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EPOCHS_M0 = 1 if DEBUG else 12\nhist0, qwk0, min0 = fit_model(m0, \"M0\", X0_tr, y_tr, X0_va, y_va, lr=1e-3,\n                              loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"],\n                              epochs=EPOCHS_M0, regression=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:28:19.988413Z","iopub.execute_input":"2026-10-05T11:28:19.989182Z","iopub.status.idle":"2026-10-05T11:28:47.224892Z","shell.execute_reply.started":"2026-10-05T11:28:19.989151Z","shell.execute_reply":"2026-10-05T11:28:47.22393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EPOCHS_M1 = 1 if DEBUG else 25\ny_tr_f, y_va_f = y_tr.astype(\"float32\"), y_va.astype(\"float32\")\nhist1, qwk1, min1 = fit_model(m1, \"M1\", X1_tr, y_tr_f, X1_va, y_va_f, lr=3e-4,\n                              loss=\"huber\", metrics=[\"mae\"],\n                              epochs=EPOCHS_M1, regression=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:29:52.960472Z","iopub.execute_input":"2026-10-05T11:29:52.960919Z","iopub.status.idle":"2026-10-05T11:31:01.865289Z","shell.execute_reply.started":"2026-10-05T11:29:52.96089Z","shell.execute_reply":"2026-10-05T11:31:01.864271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Training setup:** Adam optimiser, batch size 32, early stopping on validation loss (patience 5, best weights restored), ReduceLROnPlateau (halve the learning rate after 2 epochs without improvement).\nM0 learning rate 1e-3 (only the new head is trained); M1 learning rate 3e-4 (whole network fine-tuned, so a smaller rate is used).","metadata":{}},{"cell_type":"markdown","source":"### 6.1 Learning curves","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(12, 8))\nfor row, (name, h, q) in enumerate([(\"M0\", hist0, qwk0), (\"M1\", hist1, qwk1)]):\n    ax[row, 0].plot(h[\"loss\"], label=\"train loss\"); ax[row, 0].plot(h[\"val_loss\"], label=\"val loss\")\n    ax[row, 0].set_title(f\"{name}: loss\"); ax[row, 0].set_xlabel(\"epoch\"); ax[row, 0].legend()\n    ax[row, 1].plot(q, color=\"green\", label=\"val QWK\")\n    ax[row, 1].set_title(f\"{name}: validation QWK\"); ax[row, 1].set_xlabel(\"epoch\"); ax[row, 1].legend()\nplt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:31:08.769049Z","iopub.execute_input":"2026-10-05T11:31:08.769395Z","iopub.status.idle":"2026-10-05T11:31:09.365148Z","shell.execute_reply.started":"2026-10-05T11:31:08.76937Z","shell.execute_reply":"2026-10-05T11:31:09.364341Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> 📸 **Screenshot 6 (delete before submission):** the learning-curve figure.","metadata":{}},{"cell_type":"markdown","source":"## 7. Evaluation and diagnosis","metadata":{}},{"cell_type":"code","source":"# M0: argmax of class probabilities\np0 = m0.predict(X0_va, batch_size=64, verbose=0)\ngrade0 = p0.argmax(1)\nval_qwk0 = qwk(y_va, grade0)\n\n# M1: regression output rounded with default thresholds\np1 = m1.predict(X1_va, batch_size=64, verbose=0).ravel()\nval_qwk1 = qwk(y_va, to_grade(p1))\n\nprint(f\"M0 validation QWK: {val_qwk0:.4f}  | accuracy: {(grade0 == y_va).mean():.4f}\")\nprint(f\"M1 validation QWK: {val_qwk1:.4f}  | accuracy: {(to_grade(p1) == y_va).mean():.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:31:18.07985Z","iopub.execute_input":"2026-10-05T11:31:18.080555Z","iopub.status.idle":"2026-10-05T11:31:23.224646Z","shell.execute_reply.started":"2026-10-05T11:31:18.0805Z","shell.execute_reply":"2026-10-05T11:31:23.223773Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.1 M2: flip test-time augmentation (TTA) and optimised thresholds\nM2 uses the same trained network as M1 (no extra training). TTA averages the prediction on the original image and its flipped versions. Thresholds between grades are then tuned to maximise QWK on the validation set.\n*Note: thresholds tuned on the validation set make the M2 validation score slightly optimistic.*","metadata":{}},{"cell_type":"code","source":"def predict_tta(model, X, bs=64):\n    p = model.predict(X, batch_size=bs, verbose=0)\n    p += model.predict(X[:, :, ::-1], batch_size=bs, verbose=0)    # horizontal flip\n    p += model.predict(X[:, ::-1, :], batch_size=bs, verbose=0)    # vertical flip\n    return (p / 3).ravel()\n\np2 = predict_tta(m1, X1_va)\nthr2 = optimise_thresholds(p2, y_va)\nval_qwk2 = qwk(y_va, to_grade(p2, thr2))\nprint(\"TTA, default thresholds:\", round(qwk(y_va, to_grade(p2)), 4))\nprint(\"Optimised thresholds   :\", [round(float(t), 2) for t in thr2])\nprint(f\"M2 validation QWK: {val_qwk2:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:31:28.118413Z","iopub.execute_input":"2026-10-05T11:31:28.11907Z","iopub.status.idle":"2026-10-05T11:31:36.241896Z","shell.execute_reply.started":"2026-10-05T11:31:28.11904Z","shell.execute_reply":"2026-10-05T11:31:36.241091Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 7.2 Confusion matrix and per-class report (final model M2)","metadata":{}},{"cell_type":"code","source":"grade2 = to_grade(p2, thr2)\ncm = confusion_matrix(y_va, grade2, labels=range(5))\n\nplt.figure(figsize=(6, 5))\nplt.imshow(cm, cmap=\"Blues\"); plt.colorbar()\nplt.xticks(range(5), range(5)); plt.yticks(range(5), range(5))\nfor i in range(5):\n    for j in range(5):\n        plt.text(j, i, cm[i, j], ha=\"center\", va=\"center\",\n                 color=\"white\" if cm[i, j] > cm.max() / 2 else \"black\")\nplt.xlabel(\"Predicted grade\"); plt.ylabel(\"True grade\"); plt.title(\"M2 validation confusion matrix\")\nplt.tight_layout(); plt.show()\n\nprint(classification_report(y_va, grade2, labels=range(5), target_names=CLASS_NAMES, zero_division=0))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:31:42.152487Z","iopub.execute_input":"2026-10-05T11:31:42.152867Z","iopub.status.idle":"2026-10-05T11:31:42.38201Z","shell.execute_reply.started":"2026-10-05T11:31:42.152843Z","shell.execute_reply":"2026-10-05T11:31:42.381174Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Diagnosis (write after seeing results):**\n- *Baseline M0:* compare train loss and val loss in the curves. A large gap means overfitting; both high means underfitting.\n- *Most common error:* look at the off-diagonal cells. Neighbouring grades (1 vs 2, 2 vs 3) are usually confused because the visual difference is small, and the small classes 3 and 4 have few training examples.\n- *Why M1/M2 differ from M0:* fine-tuning lets the network adapt ImageNet features to retina lesions, augmentation reduces overfitting, regression respects the order of the grades, and tuned thresholds directly optimise QWK.\n\n> 📸 **Screenshot 7 (delete before submission):** confusion matrix and classification report.","metadata":{}},{"cell_type":"markdown","source":"## 8. Results table","metadata":{}},{"cell_type":"code","source":"def n_params(model): return f\"{trainable_params(model):,}\"\nresults = pd.DataFrame([\n    {\"Model ID\": \"M0 (Baseline)\", \"Architecture\": \"EfficientNet-B0 frozen + GAP + Dropout 0.3 + Dense(5, softmax), 224px\",\n     \"Key hyperparameters\": \"Adam 1e-3, batch 32, no augmentation, CE loss\",\n     \"Trainable params\": n_params(m0), \"Val QWK\": round(val_qwk0, 4), \"Train min\": round(min0, 1), \"Kaggle Private score\": \"fill in\"},\n    {\"Model ID\": \"M1 (Improved)\", \"Architecture\": \"EfficientNet-B0 fine-tuned + GAP + Dropout 0.4 + Dense(1), 256px, Ben Graham\",\n     \"Key hyperparameters\": \"Adam 3e-4, batch 32, flip/rotate/zoom, Huber loss, ReduceLROnPlateau\",\n     \"Trainable params\": n_params(m1), \"Val QWK\": round(val_qwk1, 4), \"Train min\": round(min1, 1), \"Kaggle Private score\": \"fill in\"},\n    {\"Model ID\": \"M2 (Improved)\", \"Architecture\": \"M1 + flip TTA + optimised thresholds\",\n     \"Key hyperparameters\": \"same network as M1, thresholds \" + str([round(float(t), 2) for t in thr2]),\n     \"Trainable params\": n_params(m1), \"Val QWK\": round(val_qwk2, 4), \"Train min\": round(min1, 1), \"Kaggle Private score\": \"fill in\"},\n])\npd.set_option(\"display.max_colwidth\", 90)\nresults","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:31:48.193177Z","iopub.execute_input":"2026-10-05T11:31:48.193511Z","iopub.status.idle":"2026-10-05T11:31:48.218348Z","shell.execute_reply.started":"2026-10-05T11:31:48.193487Z","shell.execute_reply":"2026-10-05T11:31:48.217477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"After each Kaggle submission finishes, type the Public/Private score into this table (edit the `\"fill in\"` text) so the table matches your Submissions screenshot.\n\n> 📸 **Screenshot 8 (delete before submission):** this results table.","metadata":{}},{"cell_type":"markdown","source":"## 9. Test prediction and submission file\nThe selected model (`SUBMIT_AS`) predicts the test images in chunks (to save memory) and writes `submission.csv` in the exact format of `sample_submission.csv` (`id_code`, `diagnosis` as integers 0-4).","metadata":{}},{"cell_type":"code","source":"def predict_test(kind):\n    ids = sample.id_code.values\n    if DEBUG:\n        ids = ids[:64]\n    chunks = []\n    for s in range(0, len(ids), 256):\n        X0, X1 = load_images(ids[s:s + 256], TEST_DIR)\n        if kind == \"M0\":\n            chunks.append(m0.predict(X0, batch_size=64, verbose=0).argmax(1))\n        elif kind == \"M1\":\n            chunks.append(to_grade(m1.predict(X1, batch_size=64, verbose=0)))\n        else:  # M2\n            chunks.append(to_grade(predict_tta(m1, X1), thr2))\n    return ids, np.concatenate(chunks)\n\nprint(\"Writing submission for:\", SUBMIT_AS)\nids, grades = predict_test(SUBMIT_AS)\nsubmission = pd.DataFrame({\"id_code\": ids, \"diagnosis\": grades.astype(int)})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(submission.shape); print(submission.diagnosis.value_counts().sort_index())\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:31:54.246022Z","iopub.execute_input":"2026-10-05T11:31:54.246345Z","iopub.status.idle":"2026-10-05T11:31:59.764501Z","shell.execute_reply.started":"2026-10-05T11:31:54.246323Z","shell.execute_reply":"2026-10-05T11:31:59.763401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# format check against sample_submission.csv\nassert list(submission.columns) == list(sample.columns), \"column names differ\"\nif not DEBUG:\n    assert len(submission) == len(sample) and (submission.id_code.values == sample.id_code.values).all(), \"ids/order differ\"\n    assert submission.diagnosis.between(0, 4).all(), \"grades must be 0-4\"\nprint(\"submission.csv format OK\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T11:32:05.913091Z","iopub.execute_input":"2026-10-05T11:32:05.913983Z","iopub.status.idle":"2026-10-05T11:32:05.920057Z","shell.execute_reply.started":"2026-10-05T11:32:05.913942Z","shell.execute_reply":"2026-10-05T11:32:05.91909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> 📸 **Screenshot 9 (delete before submission):** the submission.csv head and the format-check message.","metadata":{}},{"cell_type":"markdown","source":"## 10. References\n- Tan, M. and Le, Q. (2019). *EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks.* ICML.\n- Graham, B. (2015). *Kaggle Diabetic Retinopathy Detection competition report* (contrast enhancement by subtracting local average colour).\n- Keras Applications EfficientNetB0 (ImageNet weights) - https://keras.io/api/applications/efficientnet/\n- Competition: https://www.kaggle.com/c/aptos2019-blindness-detection\n- Public notebooks consulted for ideas only: add titles and links here if you use any.","metadata":{}},{"cell_type":"markdown","source":"## 11. Additional experiment: M3 (higher input resolution)\nLesions in retina images are very small, so a larger input may help. M3 keeps the M1 recipe but uses **320x320** input, a lower learning rate (2e-4), batch size 16 and up to 30 epochs. This is a separately trained model, not only a different prediction rule.","metadata":{}},{"cell_type":"code","source":"def load_one_320(args):\n    folder, i = args\n    return prep_ben(read_rgb(f\"{folder}/{i}.png\"), 320)\n\ndef load_320(ids, folder):\n    with ThreadPoolExecutor(max_workers=8) as ex:\n        return np.stack(list(ex.map(load_one_320, [(folder, i) for i in ids])))\n\nt0 = time.time()\nX3_all = load_320(train.id_code.values, TRAIN_DIR)\nX3_tr, X3_va = X3_all[idx_tr], X3_all[idx_va]\nprint(X3_all.shape, f\"| {time.time() - t0:.0f} s\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_m3():\n    base = make_backbone(320)\n    base.trainable = True\n    for l in base.layers:\n        if isinstance(l, layers.BatchNormalization):\n            l.trainable = False\n    inp = keras.Input((320, 320, 3))\n    x = layers.RandomFlip(\"horizontal_and_vertical\")(inp)\n    x = layers.RandomRotation(0.5, fill_mode=\"constant\")(x)\n    x = layers.RandomZoom(0.1)(x)\n    x = base(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.4)(x)\n    out = layers.Dense(1)(x)\n    return keras.Model(inp, out, name=\"M3_high_resolution\")\n\nm3 = build_m3()\nprint(\"M3 trainable parameters:\", f\"{trainable_params(m3):,}\")\n\nEPOCHS_M3 = 1 if DEBUG else 30\nm3.compile(optimizer=keras.optimizers.Adam(2e-4), loss=\"huber\", metrics=[\"mae\"])\nqwk_cb3 = QWKHistory(X3_va, y_va, True)\ncbs3 = [qwk_cb3,\n        keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, restore_best_weights=True),\n        keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=2, min_lr=1e-6)]\nt0 = time.time()\nh3 = m3.fit(X3_tr, y_tr.astype(\"float32\"), validation_data=(X3_va, y_va.astype(\"float32\")),\n            epochs=EPOCHS_M3, batch_size=16, callbacks=cbs3, verbose=2)\nmin3 = (time.time() - t0) / 60\nhist3, qwk3 = h3.history, qwk_cb3.values\nprint(f\"M3: {len(hist3['loss'])} epochs run, {min3:.1f} min\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(12, 4))\nax[0].plot(hist3[\"loss\"], label=\"train loss\"); ax[0].plot(hist3[\"val_loss\"], label=\"val loss\")\nax[0].set_title(\"M3: loss\"); ax[0].set_xlabel(\"epoch\"); ax[0].legend()\nax[1].plot(qwk3, color=\"green\", label=\"val QWK\")\nax[1].set_title(\"M3: validation QWK\"); ax[1].set_xlabel(\"epoch\"); ax[1].legend()\nplt.tight_layout(); plt.show()\n\np3 = m3.predict(X3_va, batch_size=32, verbose=0).ravel()\nval_qwk3 = qwk(y_va, to_grade(p3))\nprint(f\"M3 validation QWK: {val_qwk3:.4f}  | accuracy: {(to_grade(p3) == y_va).mean():.4f}\")\n\nrow = {\"Model ID\": \"M3 (Improved)\",\n       \"Architecture\": \"EfficientNet-B0 fine-tuned + GAP + Dropout 0.4 + Dense(1), 320px, Ben Graham\",\n       \"Key hyperparameters\": \"Adam 2e-4, batch 16, flip/rotate/zoom, Huber loss, ReduceLROnPlateau\",\n       \"Trainable params\": f\"{trainable_params(m3):,}\", \"Val QWK\": round(val_qwk3, 4),\n       \"Train min\": round(min3, 1), \"Kaggle Private score\": \"fill in\"}\nresults = pd.concat([results, pd.DataFrame([row])], ignore_index=True)\nresults","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if SUBMIT_AS == \"M3\":\n    ids = sample.id_code.values\n    if DEBUG:\n        ids = ids[:64]\n    chunks = []\n    for s in range(0, len(ids), 256):\n        Xc = load_320(ids[s:s + 256], TEST_DIR)\n        chunks.append(to_grade(m3.predict(Xc, batch_size=32, verbose=0)))\n    submission = pd.DataFrame({\"id_code\": ids, \"diagnosis\": np.concatenate(chunks).astype(int)})\n    submission.to_csv(\"submission.csv\", index=False)\n    print(\"Writing submission for: M3\")\n    print(submission.diagnosis.value_counts().sort_index())\n    assert list(submission.columns) == list(sample.columns), \"column names differ\"\n    if not DEBUG:\n        assert len(submission) == len(sample) and (submission.id_code.values == sample.id_code.values).all(), \"ids/order differ\"\n        assert submission.diagnosis.between(0, 4).all(), \"grades must be 0-4\"\n    print(\"submission.csv format OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 12. Additional experiment: M4 (deeper classification head)\nM4 keeps the M1 setup (256x256 input, augmentation, Huber loss) but adds **extra layers on top of the pretrained backbone**: BatchNormalization, a hidden Dense(256, ReLU) layer, a second BatchNormalization with Dropout, and a hidden Dense(64, ReLU) layer before the output. The aim is to give the model more capacity to learn from the EfficientNet features.","metadata":{}},{"cell_type":"code","source":"def build_m4():\n    base = make_backbone(256)\n    base.trainable = True\n    for l in base.layers:\n        if isinstance(l, layers.BatchNormalization):\n            l.trainable = False\n    inp = keras.Input((256, 256, 3))\n    x = layers.RandomFlip(\"horizontal_and_vertical\")(inp)\n    x = layers.RandomRotation(0.5, fill_mode=\"constant\")(x)\n    x = layers.RandomZoom(0.1)(x)\n    x = base(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.BatchNormalization()(x)                 # extra layer 1\n    x = layers.Dropout(0.4)(x)\n    x = layers.Dense(256, activation=\"relu\")(x)        # extra layer 2 (hidden)\n    x = layers.BatchNormalization()(x)                 # extra layer 3\n    x = layers.Dropout(0.3)(x)\n    x = layers.Dense(64, activation=\"relu\")(x)         # extra layer 4 (hidden)\n    out = layers.Dense(1)(x)\n    return keras.Model(inp, out, name=\"M4_deeper_head\")\n\nm4 = build_m4()\nm4.summary()\nprint(\"M4 trainable parameters:\", f\"{trainable_params(m4):,}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EPOCHS_M4 = 1 if DEBUG else 30\nhist4, qwk4, min4 = fit_model(m4, \"M4\", X1_tr, y_tr_f, X1_va, y_va_f, lr=3e-4,\n                              loss=\"huber\", metrics=[\"mae\"], epochs=EPOCHS_M4, regression=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(12, 4))\nax[0].plot(hist4[\"loss\"], label=\"train loss\"); ax[0].plot(hist4[\"val_loss\"], label=\"val loss\")\nax[0].set_title(\"M4: loss\"); ax[0].set_xlabel(\"epoch\"); ax[0].legend()\nax[1].plot(qwk4, color=\"green\", label=\"val QWK\")\nax[1].set_title(\"M4: validation QWK\"); ax[1].set_xlabel(\"epoch\"); ax[1].legend()\nplt.tight_layout(); plt.show()\n\np4 = m4.predict(X1_va, batch_size=64, verbose=0).ravel()\nval_qwk4 = qwk(y_va, to_grade(p4))\nprint(f\"M4 validation QWK: {val_qwk4:.4f}  | accuracy: {(to_grade(p4) == y_va).mean():.4f}\")\n\nrow = {\"Model ID\": \"M4 (Improved)\",\n       \"Architecture\": \"EfficientNet-B0 fine-tuned + GAP + BN + Dense(256) + BN + Dense(64) + Dense(1), 256px\",\n       \"Key hyperparameters\": \"Adam 3e-4, batch 32, flip/rotate/zoom, Huber loss, Dropout 0.4/0.3, ReduceLROnPlateau\",\n       \"Trainable params\": f\"{trainable_params(m4):,}\", \"Val QWK\": round(val_qwk4, 4),\n       \"Train min\": round(min4, 1), \"Kaggle Private score\": \"fill in\"}\nresults = pd.concat([results, pd.DataFrame([row])], ignore_index=True)\nresults","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if SUBMIT_AS == \"M4\":\n    ids = sample.id_code.values\n    if DEBUG:\n        ids = ids[:64]\n    chunks = []\n    for s in range(0, len(ids), 256):\n        _, Xc = load_images(ids[s:s + 256], TEST_DIR)\n        chunks.append(to_grade(m4.predict(Xc, batch_size=64, verbose=0)))\n    submission = pd.DataFrame({\"id_code\": ids, \"diagnosis\": np.concatenate(chunks).astype(int)})\n    submission.to_csv(\"submission.csv\", index=False)\n    print(\"Writing submission for: M4\")\n    print(submission.diagnosis.value_counts().sort_index())\n    assert list(submission.columns) == list(sample.columns), \"column names differ\"\n    if not DEBUG:\n        assert len(submission) == len(sample) and (submission.id_code.values == sample.id_code.values).all(), \"ids/order differ\"\n        assert submission.diagnosis.between(0, 4).all(), \"grades must be 0-4\"\n    print(\"submission.csv format OK\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}