{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Setup\n\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\nfrom joblib import Parallel, delayed","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:26:48.693541Z","iopub.execute_input":"2026-10-08T11:26:48.693814Z","iopub.status.idle":"2026-10-08T11:26:48.698596Z","shell.execute_reply.started":"2026-10-08T11:26:48.693794Z","shell.execute_reply":"2026-10-08T11:26:48.697757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Image size\nIMG_SIZE = 224\n\n\n# Competition dataset path\nBASE = '/kaggle/input/competitions/aptos2019-blindness-detection'\n\n\n# Find Step 2 output directory\nDATA = None\n\nfor root, dirs, files in os.walk('/kaggle/input'):\n    if 'trainval.csv' in files and 'test_holdout.csv' in files:\n        DATA = root\n        break\n\n\nprint('Step 2 data path:', DATA)\n\n\n# Check that Step 2 data was found\nif DATA is None:\n    raise FileNotFoundError(\n        'Could not find trainval.csv and test_holdout.csv under /kaggle/input'\n    )\n\n\n# Load the exact Step 2 split\ntrainval_df = pd.read_csv(f'{DATA}/trainval.csv')\ntest_df = pd.read_csv(f'{DATA}/test_holdout.csv')\n\n\n# Class names\nnames = [\n    'No DR',\n    'Mild',\n    'Moderate',\n    'Severe',\n    'Proliferative'\n]\n\n\n# Basic checks\nprint('Train/Validation images:', len(trainval_df))\nprint('Held-out test images:', len(test_df))\nprint('Number of classes:', len(names))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:27:36.155621Z","iopub.execute_input":"2026-10-08T11:27:36.155848Z","iopub.status.idle":"2026-10-08T11:27:36.197949Z","shell.execute_reply.started":"2026-10-08T11:27:36.155832Z","shell.execute_reply":"2026-10-08T11:27:36.197373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Crop, make square, and resize image\ndef crop_and_square(img, thresh=10):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n\n    mask = gray > thresh\n\n    if mask.any():\n        ys, xs = np.where(mask)\n\n        img = img[\n            ys.min():ys.max() + 1,\n            xs.min():xs.max() + 1\n        ]\n\n    h, w = img.shape[:2]\n    s = max(h, w)\n\n    canvas = np.zeros(\n        (s, s, 3),\n        dtype=img.dtype\n    )\n\n    canvas[\n        (s - h) // 2:(s - h) // 2 + h,\n        (s - w) // 2:(s - w) // 2 + w\n    ] = img\n\n    return cv2.resize(\n        canvas,\n        (IMG_SIZE, IMG_SIZE),\n        interpolation=cv2.INTER_AREA\n    )\n\n\n# Apply CLAHE enhancement\ndef clahe_enhance(img):\n    lab = cv2.cvtColor(\n        img,\n        cv2.COLOR_RGB2LAB\n    )\n\n    l, a, b = cv2.split(lab)\n\n    l = cv2.createCLAHE(\n        clipLimit=2.0,\n        tileGridSize=(8, 8)\n    ).apply(l)\n\n    return cv2.cvtColor(\n        cv2.merge((l, a, b)),\n        cv2.COLOR_LAB2RGB\n    )\n\n\n# Apply Ben Graham enhancement\ndef ben_graham(img):\n    blur = cv2.GaussianBlur(\n        img,\n        (0, 0),\n        IMG_SIZE / 30\n    )\n\n    return cv2.addWeighted(\n        img,\n        4,\n        blur,\n        -4,\n        128\n    )\n\n\n# Apply masked Ben Graham enhancement\ndef ben_graham_masked(img):\n    out = ben_graham(img)\n\n    gray = cv2.cvtColor(\n        img,\n        cv2.COLOR_RGB2GRAY\n    )\n\n    mask = (\n        gray > 8\n    ).astype(np.uint8)\n\n    mask = cv2.morphologyEx(\n        mask,\n        cv2.MORPH_CLOSE,\n        np.ones((15, 15), np.uint8)\n    )\n\n    mask = cv2.erode(\n        mask,\n        np.ones((5, 5), np.uint8),\n        iterations=3\n    )\n\n    return np.where(\n        mask[..., None] > 0,\n        out,\n        128\n    ).astype(np.uint8)\n\n\n# Load image in RGB format\ndef load_rgb(image_id):\n    image_path = f'{BASE}/train_images/{image_id}.png'\n\n    img = cv2.imread(image_path)\n\n    if img is None:\n        raise FileNotFoundError(\n            f'Could not read image: {image_path}'\n        )\n\n    return cv2.cvtColor(\n        img,\n        cv2.COLOR_BGR2RGB\n    )\n\n\nprint(\"Preprocessing functions defined successfully.\")\nprint(f\"Image size: {IMG_SIZE} × {IMG_SIZE}\")\nprint(f\"Dataset path: {BASE}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:31:23.131523Z","iopub.execute_input":"2026-10-08T11:31:23.131783Z","iopub.status.idle":"2026-10-08T11:31:23.142541Z","shell.execute_reply.started":"2026-10-08T11:31:23.131763Z","shell.execute_reply":"2026-10-08T11:31:23.141956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Additional preprocessing methods\n\ndef retina_mask(img, thresh=8):\n    \"\"\"\n    Create a mask covering the retinal region.\n    \"\"\"\n\n    gray = cv2.cvtColor(\n        img,\n        cv2.COLOR_RGB2GRAY\n    )\n\n    m = (\n        gray > thresh\n    ).astype(np.uint8)\n\n    m = cv2.morphologyEx(\n        m,\n        cv2.MORPH_CLOSE,\n        np.ones((15, 15), np.uint8)\n    )\n\n    m = cv2.erode(\n        m,\n        np.ones((5, 5), np.uint8),\n        iterations=3\n    )\n\n    return m\n\n\ndef no_enhancement(img):\n    \"\"\"\n    Control method:\n    no contrast enhancement.\n    \"\"\"\n\n    return img\n\n\ndef hist_equalize(img):\n    \"\"\"\n    Global histogram equalization\n    on the luminance channel.\n    \"\"\"\n\n    lab = cv2.cvtColor(\n        img,\n        cv2.COLOR_RGB2LAB\n    )\n\n    lab[..., 0] = cv2.equalizeHist(\n        lab[..., 0]\n    )\n\n    return cv2.cvtColor(\n        lab,\n        cv2.COLOR_LAB2RGB\n    )\n\n\ndef green_clahe(img):\n    \"\"\"\n    Apply CLAHE to the green channel.\n\n    The enhanced green channel is copied\n    into all three RGB channels so that\n    the CNN still receives 3-channel input.\n    \"\"\"\n\n    g = cv2.createCLAHE(\n        clipLimit=2.0,\n        tileGridSize=(8, 8)\n    ).apply(img[..., 1])\n\n    return np.stack(\n        [g, g, g],\n        axis=-1\n    )\n\n\ndef grayworld_clahe(img):\n    \"\"\"\n    Gray-world colour correction followed by CLAHE.\n    \"\"\"\n\n    m = retina_mask(img).astype(bool)\n\n    out = img.astype(np.float32)\n\n    if m.any():\n\n        means = out[m].mean(axis=0)\n\n        out = out * (\n            means.mean() /\n            (means + 1e-6)\n        )\n\n    out = np.clip(\n        out,\n        0,\n        255\n    ).astype(np.uint8)\n\n    return clahe_enhance(out)\n\n\ndef denoise_clahe(img):\n    \"\"\"\n    Median denoising followed by CLAHE.\n    \"\"\"\n\n    denoised = cv2.medianBlur(\n        img,\n        3\n    )\n\n    return clahe_enhance(\n        denoised\n    )\n\n\ndef clahe_sharpen(img):\n    \"\"\"\n    CLAHE followed by mild\n    unsharp masking.\n    \"\"\"\n\n    enhanced = clahe_enhance(img)\n\n    blur = cv2.GaussianBlur(\n        enhanced,\n        (0, 0),\n        2\n    )\n\n    sharpened = cv2.addWeighted(\n        enhanced,\n        1.5,\n        blur,\n        -0.5,\n        0\n    )\n\n    return sharpened\n\n\n# All preprocessing methods\nMETHODS = {\n\n    'none': no_enhancement,\n\n    'hist_eq': hist_equalize,\n\n    'clahe': clahe_enhance,\n\n    'green_clahe': green_clahe,\n\n    'grayworld_clahe': grayworld_clahe,\n\n    'denoise_clahe': denoise_clahe,\n\n    'clahe_sharpen': clahe_sharpen,\n\n    'bengraham': ben_graham_masked,\n}\n\n\nprint('Methods:', list(METHODS.keys()))\nprint('Number of methods:', len(METHODS))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:32:09.897876Z","iopub.execute_input":"2026-10-08T11:32:09.898153Z","iopub.status.idle":"2026-10-08T11:32:09.907988Z","shell.execute_reply.started":"2026-10-08T11:32:09.898133Z","shell.execute_reply":"2026-10-08T11:32:09.907292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build base images\n\ndef load_base(iid):\n    \"\"\"\n    Load an image and perform only\n    the common crop + square preprocessing.\n    \"\"\"\n\n    return crop_and_square(\n        load_rgb(iid)\n    )\n\n\ndef build_base(frame):\n    \"\"\"\n    Build base images for all image IDs.\n    \"\"\"\n\n    imgs = Parallel(\n        n_jobs=-1,\n        verbose=5\n    )(\n        delayed(load_base)(i)\n        for i in frame['id_code'].values\n    )\n\n    return np.stack(imgs)\n\n\n# Build train/validation base images\nX_base_tv = build_base(\n    trainval_df\n)\n\n\n# Build held-out test base images\nX_base_te = build_base(\n    test_df\n)\n\n\n# Labels\ny_tv = trainval_df[\n    'diagnosis'\n].values.astype(np.int64)\n\ny_te = test_df[\n    'diagnosis'\n].values.astype(np.int64)\n\n\n# Save base images\nnp.save(\n    '/kaggle/working/X_trainval_base.npy',\n    X_base_tv\n)\n\nnp.save(\n    '/kaggle/working/X_test_base.npy',\n    X_base_te\n)\n\n\n# Save labels as well\nnp.save(\n    '/kaggle/working/y_trainval.npy',\n    y_tv\n)\n\nnp.save(\n    '/kaggle/working/y_test.npy',\n    y_te\n)\n\n\nprint('Train/Validation base:', X_base_tv.shape)\nprint('Test base:', X_base_te.shape)\nprint('Train/Validation labels:', y_tv.shape)\nprint('Test labels:', y_te.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:33:08.818623Z","iopub.execute_input":"2026-10-08T11:33:08.818851Z","iopub.status.idle":"2026-10-08T11:35:48.722824Z","shell.execute_reply.started":"2026-10-08T11:33:08.818835Z","shell.execute_reply":"2026-10-08T11:35:48.721824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify compatibility with Step 2 outputs\n\nprint('Loading Step 2 CLAHE output...')\n\nsaved_c = np.load(\n    f'{DATA}/X_trainval_clahe.npy',\n    mmap_mode='r'\n)\n\n\nprint('Loading Step 2 Ben Graham output...')\n\nsaved_b = np.load(\n    f'{DATA}/X_trainval_bengraham.npy',\n    mmap_mode='r'\n)\n\n\n# Check first 30 images\nok_c = all(\n    np.array_equal(\n        clahe_enhance(X_base_tv[i]),\n        saved_c[i]\n    )\n    for i in range(30)\n)\n\n\nok_b = all(\n    np.array_equal(\n        ben_graham_masked(X_base_tv[i]),\n        saved_b[i]\n    )\n    for i in range(30)\n)\n\n\nprint(\n    'CLAHE matches Step 2:',\n    ok_c\n)\n\nprint(\n    'Ben Graham matches Step 2:',\n    ok_b\n)\n\n\n# Stop if either pipeline does not match\nif not ok_c or not ok_b:\n    raise RuntimeError(\n        'Step 2 consistency check failed. '\n        'Do not continue until the preprocessing functions are verified.'\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:36:44.548552Z","iopub.execute_input":"2026-10-08T11:36:44.548836Z","iopub.status.idle":"2026-10-08T11:36:44.843533Z","shell.execute_reply.started":"2026-10-08T11:36:44.548818Z","shell.execute_reply":"2026-10-08T11:36:44.842678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visual comparison of all preprocessing methods\n\nsample = [\n    trainval_df.index[\n        trainval_df.diagnosis == c\n    ][0]\n    for c in range(5)\n]\n\n\nfig, axes = plt.subplots(\n    5,\n    8,\n    figsize=(24, 15)\n)\n\n\nfor r, i in enumerate(sample):\n\n    for c, (method_name, method_fn) in enumerate(\n        METHODS.items()\n    ):\n\n        processed = method_fn(\n            X_base_tv[i]\n        )\n\n        axes[r][c].imshow(\n            processed\n        )\n\n        axes[r][c].axis('off')\n\n        axes[r][c].set_title(\n            f'{names[r]}: {method_name}',\n            fontsize=8\n        )\n\n\nplt.tight_layout()\n\n\nplt.savefig(\n    '/kaggle/working/preprocessing_8_methods.png',\n    dpi=110,\n    bbox_inches='tight'\n)\n\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:37:37.678194Z","iopub.execute_input":"2026-10-08T11:37:37.678511Z","iopub.status.idle":"2026-10-08T11:37:43.427621Z","shell.execute_reply.started":"2026-10-08T11:37:37.678494Z","shell.execute_reply":"2026-10-08T11:37:43.426822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Numerical comparison of preprocessing methods\n\nrng = np.random.RandomState(0)\n\n\n# Select 300 images\nsample_idx = rng.choice(\n    len(X_base_tv),\n    300,\n    replace=False\n)\n\n\n# Create the same retina mask for every method\nmasks = {\n    i: retina_mask(\n        X_base_tv[i]\n    ).astype(bool)\n\n    for i in sample_idx\n}\n\n\ndef img_stats(img, mask):\n    \"\"\"\n    Calculate image statistics\n    inside the retinal region.\n    \"\"\"\n\n    # Convert RGB to grayscale\n    gray = cv2.cvtColor(\n        img,\n        cv2.COLOR_RGB2GRAY\n    )\n\n\n    # Retinal pixels only\n    px = gray[mask]\n\n\n    # Histogram\n    p = (\n        np.bincount(\n            px,\n            minlength=256\n        )\n        / px.size\n    )\n\n\n    # Entropy\n    entropy = -(\n        p[p > 0] *\n        np.log2(p[p > 0])\n    ).sum()\n\n\n    # Laplacian variance\n    # used as a simple sharpness measure\n    lap = cv2.Laplacian(\n        gray,\n        cv2.CV_64F\n    )[mask]\n\n\n    sharpness = lap.var()\n\n\n    # Return statistics\n    return (\n        px.std(),\n        sharpness,\n        entropy,\n        px.mean()\n    )\n\n\nrows = []\n\n\nfor method_name, method_fn in METHODS.items():\n\n    values = np.array([\n        img_stats(\n            method_fn(X_base_tv[i]),\n            masks[i]\n        )\n\n        for i in sample_idx\n\n        if masks[i].any()\n    ])\n\n\n    rows.append({\n\n        'method': method_name,\n\n        'contrast': values[:, 0].mean(),\n\n        'sharpness': values[:, 1].mean(),\n\n        'entropy': values[:, 2].mean(),\n\n        'brightness_spread': values[:, 3].std()\n    })\n\n\n# Create results table\nstats_df = pd.DataFrame(\n    rows\n).set_index(\n    'method'\n).round(2)\n\n\n# Save results\nstats_df.to_csv(\n    '/kaggle/working/step2b_image_stats.csv'\n)\n\n\n# Display results\nstats_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:39:21.792146Z","iopub.execute_input":"2026-10-08T11:39:21.79246Z","iopub.status.idle":"2026-10-08T11:39:26.190756Z","shell.execute_reply.started":"2026-10-08T11:39:21.792439Z","shell.execute_reply":"2026-10-08T11:39:26.190148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display preprocessing statistics\n\nprint('Preprocessing comparison on 300 randomly selected images')\nprint('=' * 65)\n\ndisplay(\n    stats_df\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:40:53.898112Z","iopub.execute_input":"2026-10-08T11:40:53.898426Z","iopub.status.idle":"2026-10-08T11:40:53.909414Z","shell.execute_reply.started":"2026-10-08T11:40:53.898408Z","shell.execute_reply":"2026-10-08T11:40:53.90888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Relative comparison of image statistics\n\nprint('Highest contrast:')\nprint(\n    stats_df['contrast'].idxmax(),\n    '→',\n    stats_df['contrast'].max()\n)\n\n\nprint('\\nHighest sharpness:')\nprint(\n    stats_df['sharpness'].idxmax(),\n    '→',\n    stats_df['sharpness'].max()\n)\n\n\nprint('\\nHighest entropy:')\nprint(\n    stats_df['entropy'].idxmax(),\n    '→',\n    stats_df['entropy'].max()\n)\n\n\nprint('\\nLowest brightness spread:')\nprint(\n    stats_df['brightness_spread'].idxmin(),\n    '→',\n    stats_df['brightness_spread'].min()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T11:42:37.67112Z","iopub.execute_input":"2026-10-08T11:42:37.671407Z","iopub.status.idle":"2026-10-08T11:42:37.683197Z","shell.execute_reply.started":"2026-10-08T11:42:37.671389Z","shell.execute_reply":"2026-10-08T11:42:37.682463Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## More Preprocessing Conclusions\n\n- The exact Step 2 train/validation and held-out test split was reused: **3,112 train+validation images** and **550 held-out test images**, with the test set kept untouched for final evaluation.\n\n- A common base pipeline was created using **black-border cropping, square padding and resizing to 224 × 224 × 3**, allowing all preprocessing methods to be compared from the same standardized images.\n\n- Eight preprocessing methods were evaluated: **None, Histogram Equalization, CLAHE, Green-channel CLAHE, Gray-world + CLAHE, Denoising + CLAHE, CLAHE + Sharpening, and Ben Graham**.\n\n- The existing Step 2 preprocessing outputs were verified against the new pipeline. **CLAHE matches Step 2: True** and **Ben Graham matches Step 2: True**, confirming consistency between the two preprocessing pipelines.\n\n- A visual and numerical comparison was performed on **300 randomly selected train/validation images** using contrast, sharpness, entropy and brightness-spread measurements.\n\n- **Histogram Equalization** produced the highest measured contrast, sharpness and entropy, while **Ben Graham** produced the lowest brightness spread. However, high sharpness and entropy can also result from amplified noise or artifacts.\n\n- Visual inspection showed that **Histogram Equalization produced blotchy over-brightening and speckle noise**, while **Gray-world + CLAHE introduced colour casts and edge artifacts** in several images.\n\n- **Green-channel CLAHE and Ben Graham** provided clearer visualization of retinal vessels and exudates, while **Denoising + CLAHE** reduced fine-detail sharpness.\n\n- Based on the visual and image-level screening, the methods shortlisted for Step 4 are **None (control), CLAHE, Green-channel CLAHE and Ben Graham**. The other methods were excluded because of visible artifacts or because they were too similar to existing methods, **not because of classification results**.\n\n- The image-level statistics are only preliminary screening indicators. The final preprocessing method will be selected based on **classification performance**, primarily macro F1-score, per-class recall and quadratic weighted kappa.\n\n- The shortlisted preprocessing methods will be compared using the **same custom CNN, train/validation split, augmentation, class weights, optimizer and training procedure** to ensure a fair comparison.\n\n- The **550-image held-out test set will remain untouched** until the final preprocessing method and model have been selected.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}