{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nimport numpy as np\n\nprint(\"PyTorch:\", torch.__version__)\nprint(\"CUDA available:\", torch.cuda.is_available())\n\nif torch.cuda.is_available():\n    print(\"GPU:\", torch.cuda.get_device_name(0))\n    print(\"GPU memory:\",\n          round(torch.cuda.get_device_properties(0).total_memory / 1024**3, 2),\n          \"GB\")\n\nprint(\"\\nKaggle input:\")\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n    level = root.replace(\"/kaggle/input\", \"\").count(os.sep)\n    if level <= 2:\n        print(root)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-08T18:57:57.072874Z","iopub.execute_input":"2026-10-08T18:57:57.073134Z","iopub.status.idle":"2026-10-08T18:57:57.087102Z","shell.execute_reply.started":"2026-10-08T18:57:57.073116Z","shell.execute_reply":"2026-10-08T18:57:57.086473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom PIL import Image\n\nprint(\"Libraries imported successfully.\")\n\n# Original EyePACS Kaggle competition data\nEYE_PATH = \"/kaggle/input/competitions/diabetic-retinopathy-detection\"\n\n# Working directory\nWORK_DIR = \"/kaggle/working\"\n\n# Paths we will use later\nPROCESSED_224 = os.path.join(WORK_DIR, \"eyepacs_processed\")\nGRAHAM_384 = os.path.join(WORK_DIR, \"eyepacs_graham_384\")\nCHECKPOINT_DIR = os.path.join(WORK_DIR, \"checkpoints\")\n\nos.makedirs(CHECKPOINT_DIR, exist_ok=True)\n\nprint(\"EyePACS path:\", EYE_PATH)\nprint(\"Working path:\", WORK_DIR)\nprint(\"Checkpoint path:\", CHECKPOINT_DIR)\n\n# Verify the original EyePACS files exist\nprint(\"\\nEyePACS contents:\")\nfor item in sorted(os.listdir(EYE_PATH)):\n    print(\" -\", item)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T18:58:00.791928Z","iopub.execute_input":"2026-10-08T18:58:00.792243Z","iopub.status.idle":"2026-10-08T18:58:00.799839Z","shell.execute_reply.started":"2026-10-08T18:58:00.792216Z","shell.execute_reply":"2026-10-08T18:58:00.799018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import zipfile\nimport pandas as pd\nimport os\n\nlabels_zip = os.path.join(\n    EYE_PATH,\n    \"trainLabels.csv.zip\"\n)\n\nprint(\"Labels file exists:\", os.path.exists(labels_zip))\n\nwith zipfile.ZipFile(labels_zip, \"r\") as z:\n    print(\"Files inside:\", z.namelist())\n\n    with z.open(\"trainLabels.csv\") as f:\n        metadata = pd.read_csv(f)\n\nprint(\"\\nMetadata shape:\", metadata.shape)\nprint(\"\\nFirst 5 rows:\")\ndisplay(metadata.head())\n\nprint(\"\\nClass distribution:\")\nprint(metadata[\"level\"].value_counts().sort_index())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T18:58:55.016439Z","iopub.execute_input":"2026-10-08T18:58:55.016743Z","iopub.status.idle":"2026-10-08T18:58:55.112548Z","shell.execute_reply.started":"2026-10-08T18:58:55.016701Z","shell.execute_reply":"2026-10-08T18:58:55.112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata = metadata.rename(columns={\"level\": \"label\"})\n\nmetadata[\"patient_id\"] = (\n    metadata[\"image\"]\n    .str.rsplit(\"_\", n=1)\n    .str[0]\n)\n\nmetadata[\"eye\"] = (\n    metadata[\"image\"]\n    .str.rsplit(\"_\", n=1)\n    .str[1]\n)\n\nmetadata = metadata[\n    [\"patient_id\", \"image\", \"eye\", \"label\"]\n].copy()\n\nprint(\"Total images:\", len(metadata))\nprint(\"Unique patients:\", metadata[\"patient_id\"].nunique())\n\nprint(\"\\nImages per patient:\")\nprint(metadata.groupby(\"patient_id\").size().value_counts().sort_index())\n\nprint(\"\\nEyes per patient:\")\nprint(metadata.groupby(\"patient_id\")[\"eye\"].nunique().value_counts().sort_index())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T18:59:11.22586Z","iopub.execute_input":"2026-10-08T18:59:11.226671Z","iopub.status.idle":"2026-10-08T18:59:11.426563Z","shell.execute_reply.started":"2026-10-08T18:59:11.226644Z","shell.execute_reply":"2026-10-08T18:59:11.425967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nSEED = 42\n\n# One label per patient = maximum severity across both eyes\npatient_df = (\n    metadata.groupby(\"patient_id\")[\"label\"]\n    .max()\n    .reset_index()\n    .rename(columns={\"label\": \"patient_label\"})\n)\n\nprint(\"Patient distribution:\")\nprint(patient_df[\"patient_label\"].value_counts().sort_index())\n\n# 70% train, 30% temporary\ntrain_patients, temp_patients = train_test_split(\n    patient_df,\n    test_size=0.30,\n    stratify=patient_df[\"patient_label\"],\n    random_state=SEED\n)\n\n# 15% validation, 15% test\nval_patients, test_patients = train_test_split(\n    temp_patients,\n    test_size=0.50,\n    stratify=temp_patients[\"patient_label\"],\n    random_state=SEED\n)\n\ntrain_df = metadata[\n    metadata[\"patient_id\"].isin(train_patients[\"patient_id\"])\n].copy()\n\nval_df = metadata[\n    metadata[\"patient_id\"].isin(val_patients[\"patient_id\"])\n].copy()\n\ntest_df = metadata[\n    metadata[\"patient_id\"].isin(test_patients[\"patient_id\"])\n].copy()\n\nprint(\"\\n===== SPLIT =====\")\nprint(\"Train      :\", len(train_df), \"images |\",\n      train_df[\"patient_id\"].nunique(), \"patients\")\n\nprint(\"Validation :\", len(val_df), \"images |\",\n      val_df[\"patient_id\"].nunique(), \"patients\")\n\nprint(\"Test       :\", len(test_df), \"images |\",\n      test_df[\"patient_id\"].nunique(), \"patients\")\n\n# Leakage check\ntrain_ids = set(train_df[\"patient_id\"])\nval_ids = set(val_df[\"patient_id\"])\ntest_ids = set(test_df[\"patient_id\"])\n\nprint(\"\\n===== LEAKAGE CHECK =====\")\nprint(\"Train ∩ Validation:\", len(train_ids & val_ids))\nprint(\"Train ∩ Test      :\", len(train_ids & test_ids))\nprint(\"Validation ∩ Test :\", len(val_ids & test_ids))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T18:59:27.619037Z","iopub.execute_input":"2026-10-08T18:59:27.61937Z","iopub.status.idle":"2026-10-08T18:59:28.336677Z","shell.execute_reply.started":"2026-10-08T18:59:27.619339Z","shell.execute_reply":"2026-10-08T18:59:28.335542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport io\nimport zipfile\n\nclass SplitFile:\n    def __init__(self, parts):\n        self.parts = [open(p, \"rb\") for p in parts]\n        self.sizes = [os.path.getsize(p) for p in parts]\n\n        self.offsets = []\n        total = 0\n\n        for size in self.sizes:\n            self.offsets.append(total)\n            total += size\n\n        self.total_size = total\n        self.position = 0\n\n    def seek(self, offset, whence=0):\n        if whence == 0:\n            self.position = offset\n        elif whence == 1:\n            self.position += offset\n        elif whence == 2:\n            self.position = self.total_size + offset\n\n        self.position = max(0, min(self.position, self.total_size))\n        return self.position\n\n    def tell(self):\n        return self.position\n\n    def seekable(self):\n        return True\n\n    def readable(self):\n        return True\n\n    def writable(self):\n        return False\n\n    def read(self, size=-1):\n        if size is None or size < 0:\n            size = self.total_size - self.position\n\n        if size == 0 or self.position >= self.total_size:\n            return b\"\"\n\n        remaining = size\n        output = bytearray()\n\n        while remaining > 0 and self.position < self.total_size:\n\n            part_index = None\n\n            for i, start in enumerate(self.offsets):\n                if start <= self.position < start + self.sizes[i]:\n                    part_index = i\n                    break\n\n            if part_index is None:\n                break\n\n            local_offset = self.position - self.offsets[part_index]\n            available = self.sizes[part_index] - local_offset\n            to_read = min(remaining, available)\n\n            self.parts[part_index].seek(local_offset)\n            chunk = self.parts[part_index].read(to_read)\n\n            output.extend(chunk)\n\n            self.position += len(chunk)\n            remaining -= len(chunk)\n\n            if len(chunk) == 0:\n                break\n\n        return bytes(output)\n\n    def close(self):\n        for f in self.parts:\n            f.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:00:16.906531Z","iopub.execute_input":"2026-10-08T19:00:16.906991Z","iopub.status.idle":"2026-10-08T19:00:16.918431Z","shell.execute_reply.started":"2026-10-08T19:00:16.906967Z","shell.execute_reply":"2026-10-08T19:00:16.917697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_parts = [\n    os.path.join(EYE_PATH, f\"train.zip.{i:03d}\")\n    for i in range(1, 6)\n]\n\nprint(\"Train archive parts:\")\n\nfor p in train_parts:\n    print(\n        os.path.basename(p),\n        \"|\",\n        round(os.path.getsize(p) / (1024**3), 2),\n        \"GB\"\n    )\n\ntrain_split = SplitFile(train_parts)\n\ntrain_zip = zipfile.ZipFile(train_split, \"r\")\n\nprint(\"\\nVirtual ZIP opened successfully.\")\nprint(\"Number of ZIP entries:\", len(train_zip.namelist()))\n\nprint(\"\\nFirst few entries:\")\nfor name in train_zip.namelist()[:10]:\n    print(name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:01:03.808314Z","iopub.execute_input":"2026-10-08T19:01:03.808892Z","iopub.status.idle":"2026-10-08T19:01:03.995313Z","shell.execute_reply.started":"2026-10-08T19:01:03.808873Z","shell.execute_reply":"2026-10-08T19:01:03.994702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\n\ntest_image_name = \"train/10003_left.jpeg\"\n\nprint(\"Checking:\", test_image_name)\n\nwith train_zip.open(test_image_name) as f:\n    image_bytes = f.read()\n\nimage = Image.open(io.BytesIO(image_bytes)).convert(\"RGB\")\n\nprint(\"Image loaded successfully!\")\nprint(\"Size:\", image.size)\nprint(\"Mode:\", image.mode)\nprint(\"Bytes:\", len(image_bytes))\n\ndisplay(image.resize((400, 400)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:01:16.973093Z","iopub.execute_input":"2026-10-08T19:01:16.973438Z","iopub.status.idle":"2026-10-08T19:01:17.320543Z","shell.execute_reply.started":"2026-10-08T19:01:16.973411Z","shell.execute_reply":"2026-10-08T19:01:17.319812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_name = train_df.iloc[0][\"image\"]\nzip_name = f\"train/{sample_name}.jpeg\"\n\nprint(\"Patient:\", train_df.iloc[0][\"patient_id\"])\nprint(\"Image:\", sample_name)\nprint(\"ZIP path:\", zip_name)\n\nwith train_zip.open(zip_name) as f:\n    image_bytes = f.read()\n\nimage = Image.open(io.BytesIO(image_bytes)).convert(\"RGB\")\n\nprint(\"Original resolution:\", image.size)\nprint(\"Mode:\", image.mode)\n\ndisplay(image.resize((400, 400)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:01:30.540588Z","iopub.execute_input":"2026-10-08T19:01:30.540878Z","iopub.status.idle":"2026-10-08T19:01:30.863687Z","shell.execute_reply.started":"2026-10-08T19:01:30.54086Z","shell.execute_reply":"2026-10-08T19:01:30.863074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport numpy as np\nfrom PIL import Image\n\ndef graham_preprocess(pil_image, output_size=384):\n    \"\"\"\n    Graham-style retinal preprocessing:\n    1. Find retinal/non-black region\n    2. Crop around the retina\n    3. Resize\n    4. Local illumination/high-boost correction\n    5. Restore black background\n    \"\"\"\n\n    img = np.array(pil_image.convert(\"RGB\"))\n\n    # Create grayscale image\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n\n    # Detect non-black retinal region\n    mask = gray > 10\n\n    if not np.any(mask):\n        return Image.fromarray(img).resize(\n            (output_size, output_size),\n            Image.Resampling.LANCZOS\n        )\n\n    # Find bounding box\n    ys, xs = np.where(mask)\n\n    x1, x2 = xs.min(), xs.max()\n    y1, y2 = ys.min(), ys.max()\n\n    # Crop retinal region\n    cropped = img[y1:y2 + 1, x1:x2 + 1]\n\n    # Resize\n    cropped = cv2.resize(\n        cropped,\n        (output_size, output_size),\n        interpolation=cv2.INTER_AREA\n    )\n\n    # Local illumination correction\n    blur = cv2.GaussianBlur(\n        cropped,\n        (0, 0),\n        sigmaX=10\n    )\n\n    enhanced = cv2.addWeighted(\n        cropped,\n        4,\n        blur,\n        -4,\n        128\n    )\n\n    # Resize original mask\n    cropped_gray = cv2.cvtColor(cropped, cv2.COLOR_RGB2GRAY)\n\n    mask_resized = (\n        cropped_gray > 10\n    ).astype(np.uint8) * 255\n\n    # Slightly smooth mask\n    mask_resized = cv2.GaussianBlur(\n        mask_resized,\n        (5, 5),\n        0\n    )\n\n    # Restore black background\n    enhanced = np.where(\n        mask_resized[..., None] > 10,\n        enhanced,\n        0\n    )\n\n    enhanced = np.clip(enhanced, 0, 255).astype(np.uint8)\n\n    return Image.fromarray(enhanced)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:03:22.425414Z","iopub.execute_input":"2026-10-08T19:03:22.425685Z","iopub.status.idle":"2026-10-08T19:03:22.923189Z","shell.execute_reply.started":"2026-10-08T19:03:22.425667Z","shell.execute_reply":"2026-10-08T19:03:22.922209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nsample_images = [\n    \"10_left\",\n    \"15_left\",\n    \"10003_left\",\n    \"10003_right\"\n]\n\nfig, axes = plt.subplots(\n    len(sample_images),\n    2,\n    figsize=(10, 18)\n)\n\nfor row, image_name in enumerate(sample_images):\n\n    zip_name = f\"train/{image_name}.jpeg\"\n\n    with train_zip.open(zip_name) as f:\n        original_bytes = f.read()\n\n    original = Image.open(\n        io.BytesIO(original_bytes)\n    ).convert(\"RGB\")\n\n    processed = graham_preprocess(\n        original,\n        output_size=384\n    )\n\n    axes[row, 0].imshow(original)\n    axes[row, 0].set_title(\n        f\"{image_name} — Original\"\n    )\n    axes[row, 0].axis(\"off\")\n\n    axes[row, 1].imshow(processed)\n    axes[row, 1].set_title(\n        f\"{image_name} — Graham 384\"\n    )\n    axes[row, 1].axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:03:35.644602Z","iopub.execute_input":"2026-10-08T19:03:35.645203Z","iopub.status.idle":"2026-10-08T19:03:40.742571Z","shell.execute_reply.started":"2026-10-08T19:03:35.645182Z","shell.execute_reply":"2026-10-08T19:03:40.741824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"processed = graham_preprocess(\n    original,\n    output_size=384\n)\n\nprint(\"Processed size:\", processed.size)\nprint(\"Processed mode:\", processed.mode)\n\nprocessed_array = np.array(processed)\n\nprint(\"Array shape:\", processed_array.shape)\nprint(\"Min pixel:\", processed_array.min())\nprint(\"Max pixel:\", processed_array.max())\nprint(\"Mean pixel:\", processed_array.mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:04:05.9537Z","iopub.execute_input":"2026-10-08T19:04:05.954136Z","iopub.status.idle":"2026-10-08T19:04:06.146118Z","shell.execute_reply.started":"2026-10-08T19:04:05.954101Z","shell.execute_reply":"2026-10-08T19:04:06.145397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_indices = [0, 1, 2, 3]\n\nprint(\"Checking 4 training images...\\n\")\n\nfor idx in sample_indices:\n\n    image_name = train_df.iloc[idx][\"image\"]\n    zip_name = f\"train/{image_name}.jpeg\"\n\n    with train_zip.open(zip_name) as f:\n        image_bytes = f.read()\n\n    original = Image.open(\n        io.BytesIO(image_bytes)\n    ).convert(\"RGB\")\n\n    processed = graham_preprocess(\n        original,\n        output_size=384\n    )\n\n    arr = np.array(processed)\n\n    print(\n        image_name,\n        \"| original:\", original.size,\n        \"| processed:\", processed.size,\n        \"| min:\", arr.min(),\n        \"| max:\", arr.max(),\n        \"| NaN:\", np.isnan(arr).any()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:04:21.642044Z","iopub.execute_input":"2026-10-08T19:04:21.642318Z","iopub.status.idle":"2026-10-08T19:04:22.405105Z","shell.execute_reply.started":"2026-10-08T19:04:21.642299Z","shell.execute_reply":"2026-10-08T19:04:22.404392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport io\nimport time\nfrom PIL import Image\n\nGRAHAM_384 = \"/kaggle/working/eyepacs_graham_384\"\n\nfor split in [\"train\", \"validation\", \"test\"]:\n    os.makedirs(\n        os.path.join(GRAHAM_384, split),\n        exist_ok=True\n    )\n\nprint(\"Output directory:\", GRAHAM_384)\n\nfor split in [\"train\", \"validation\", \"test\"]:\n    path = os.path.join(GRAHAM_384, split)\n    print(\n        split,\n        \"directory exists:\",\n        os.path.exists(path)\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:07:47.942473Z","iopub.execute_input":"2026-10-08T19:07:47.942748Z","iopub.status.idle":"2026-10-08T19:07:47.949312Z","shell.execute_reply.started":"2026-10-08T19:07:47.942709Z","shell.execute_reply":"2026-10-08T19:07:47.948775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_split(\n    dataframe,\n    split_name,\n    zip_file,\n    output_dir,\n    is_train=True\n):\n    split_output = os.path.join(\n        output_dir,\n        split_name\n    )\n\n    os.makedirs(split_output, exist_ok=True)\n\n    total = len(dataframe)\n    processed = 0\n    skipped = 0\n    failed = 0\n\n    start_time = time.time()\n\n    print(f\"\\n===== {split_name.upper()} =====\")\n    print(\"Total:\", total)\n\n    for i, row in enumerate(\n        dataframe.itertuples(index=False),\n        start=1\n    ):\n\n        image_name = row.image\n        output_path = os.path.join(\n            split_output,\n            f\"{image_name}.jpg\"\n        )\n\n        # IMPORTANT:\n        # Skip images that were already successfully processed\n        if os.path.exists(output_path):\n            skipped += 1\n            continue\n\n        zip_name = f\"train/{image_name}.jpeg\"\n\n        try:\n            with zip_file.open(zip_name) as f:\n                image_bytes = f.read()\n\n            original = Image.open(\n                io.BytesIO(image_bytes)\n            ).convert(\"RGB\")\n\n            processed_image = graham_preprocess(\n                original,\n                output_size=384\n            )\n\n            processed_image.save(\n                output_path,\n                format=\"JPEG\",\n                quality=95\n            )\n\n            processed += 1\n\n        except Exception as e:\n            failed += 1\n\n            print(\n                f\"\\nFAILED: {image_name}\"\n                f\"\\nError: {e}\"\n            )\n\n        # Progress every 500 images\n        if i % 500 == 0:\n\n            elapsed = time.time() - start_time\n\n            print(\n                f\"{split_name}: \"\n                f\"{i}/{total} | \"\n                f\"new={processed} | \"\n                f\"skipped={skipped} | \"\n                f\"failed={failed} | \"\n                f\"time={elapsed/60:.1f} min\"\n            )\n\n    elapsed = time.time() - start_time\n\n    print(f\"\\n{split_name} COMPLETE\")\n    print(\"Processed:\", processed)\n    print(\"Skipped:\", skipped)\n    print(\"Failed:\", failed)\n    print(\"Time:\", round(elapsed / 60, 2), \"minutes\")\n\n    return {\n        \"processed\": processed,\n        \"skipped\": skipped,\n        \"failed\": failed\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:08:21.28882Z","iopub.execute_input":"2026-10-08T19:08:21.289422Z","iopub.status.idle":"2026-10-08T19:08:21.297484Z","shell.execute_reply.started":"2026-10-08T19:08:21.289395Z","shell.execute_reply":"2026-10-08T19:08:21.296545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_result = process_split(\n    dataframe=train_df,\n    split_name=\"train\",\n    zip_file=train_zip,\n    output_dir=GRAHAM_384,\n    is_train=True\n)\n\nprint(\"\\nTRAIN RESULT:\")\nprint(train_result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:09:01.63464Z","iopub.execute_input":"2026-10-08T19:09:01.634963Z","iopub.status.idle":"2026-10-08T19:38:59.348023Z","shell.execute_reply.started":"2026-10-08T19:09:01.634938Z","shell.execute_reply":"2026-10-08T19:38:59.345843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Current split:\")\nprint(\"Train:\", len(train_df), \"images |\", train_df[\"patient_id\"].nunique(), \"patients\")\nprint(\"Val  :\", len(val_df), \"images |\", val_df[\"patient_id\"].nunique(), \"patients\")\nprint(\"Test :\", len(test_df), \"images |\", test_df[\"patient_id\"].nunique(), \"patients\")\n\nprint(\"\\nTotal:\", len(train_df) + len(val_df) + len(test_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:39:15.894093Z","iopub.execute_input":"2026-10-08T19:39:15.895428Z","iopub.status.idle":"2026-10-08T19:39:15.916031Z","shell.execute_reply.started":"2026-10-08T19:39:15.895398Z","shell.execute_reply":"2026-10-08T19:39:15.914787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nprint(\"Searching /kaggle/working for split files...\\n\")\n\nfor root, dirs, files in os.walk(\"/kaggle/working\"):\n    for f in files:\n        if f in [\n            \"train_split.csv\",\n            \"val_split.csv\",\n            \"test_split.csv\"\n        ]:\n            print(os.path.join(root, f))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:40:31.237898Z","iopub.execute_input":"2026-10-08T19:40:31.238156Z","iopub.status.idle":"2026-10-08T19:40:31.252842Z","shell.execute_reply.started":"2026-10-08T19:40:31.238137Z","shell.execute_reply":"2026-10-08T19:40:31.251691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FINAL PATIENT-LEVEL SPLIT VERIFICATION\n# ============================================================\n\ntrain_ids = set(train_df[\"patient_id\"])\nval_ids = set(val_df[\"patient_id\"])\ntest_ids = set(test_df[\"patient_id\"])\n\nprint(\"===== PATIENT COUNTS =====\")\nprint(\"Train:\", len(train_ids))\nprint(\"Val  :\", len(val_ids))\nprint(\"Test :\", len(test_ids))\nprint(\"Total:\", len(train_ids | val_ids | test_ids))\n\nprint(\"\\n===== IMAGE COUNTS =====\")\nprint(\"Train:\", len(train_df))\nprint(\"Val  :\", len(val_df))\nprint(\"Test :\", len(test_df))\nprint(\"Total:\", len(train_df) + len(val_df) + len(test_df))\n\nprint(\"\\n===== LEAKAGE CHECK =====\")\nprint(\"Train ∩ Val :\", len(train_ids & val_ids))\nprint(\"Train ∩ Test:\", len(train_ids & test_ids))\nprint(\"Val ∩ Test  :\", len(val_ids & test_ids))\n\nassert len(train_ids & val_ids) == 0\nassert len(train_ids & test_ids) == 0\nassert len(val_ids & test_ids) == 0\n\nassert len(train_df) + len(val_df) + len(test_df) == 35126\n\nprint(\"\\n✅ Patient-level split verified.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:41:05.678529Z","iopub.execute_input":"2026-10-08T19:41:05.678849Z","iopub.status.idle":"2026-10-08T19:41:05.692591Z","shell.execute_reply.started":"2026-10-08T19:41:05.67883Z","shell.execute_reply":"2026-10-08T19:41:05.691894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SAVE THE EXACT SPLIT\n# ============================================================\n\nSPLIT_DIR = \"/kaggle/working/final_metadata\"\nos.makedirs(SPLIT_DIR, exist_ok=True)\n\ntrain_df.to_csv(\n    os.path.join(SPLIT_DIR, \"train_split.csv\"),\n    index=False\n)\n\nval_df.to_csv(\n    os.path.join(SPLIT_DIR, \"val_split.csv\"),\n    index=False\n)\n\ntest_df.to_csv(\n    os.path.join(SPLIT_DIR, \"test_split.csv\"),\n    index=False\n)\n\npatient_df.to_csv(\n    os.path.join(SPLIT_DIR, \"patient_labels.csv\"),\n    index=False\n)\n\nprint(\"Saved files:\")\n\nfor f in sorted(os.listdir(SPLIT_DIR)):\n    path = os.path.join(SPLIT_DIR, f)\n    print(\n        f,\n        \"|\",\n        round(os.path.getsize(path) / 1024, 2),\n        \"KB\"\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:41:35.963092Z","iopub.execute_input":"2026-10-08T19:41:35.963332Z","iopub.status.idle":"2026-10-08T19:41:36.034705Z","shell.execute_reply.started":"2026-10-08T19:41:35.963315Z","shell.execute_reply":"2026-10-08T19:41:36.033906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SPLIT SUMMARY\n# ============================================================\n\nsummary = pd.DataFrame({\n    \"split\": [\"train\", \"validation\", \"test\"],\n    \"patients\": [\n        train_df[\"patient_id\"].nunique(),\n        val_df[\"patient_id\"].nunique(),\n        test_df[\"patient_id\"].nunique()\n    ],\n    \"images\": [\n        len(train_df),\n        len(val_df),\n        len(test_df)\n    ]\n})\n\nprint(summary.to_string(index=False))\n\nprint(\"\\nClass distribution by image:\")\nprint(\n    pd.DataFrame({\n        \"train\": train_df[\"label\"].value_counts().sort_index(),\n        \"validation\": val_df[\"label\"].value_counts().sort_index(),\n        \"test\": test_df[\"label\"].value_counts().sort_index()\n    }).fillna(0).astype(int)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:42:38.61682Z","iopub.execute_input":"2026-10-08T19:42:38.617418Z","iopub.status.idle":"2026-10-08T19:42:38.630367Z","shell.execute_reply.started":"2026-10-08T19:42:38.6174Z","shell.execute_reply":"2026-10-08T19:42:38.629546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_output = os.path.join(\n    GRAHAM_384,\n    \"train\"\n)\n\ntrain_files = [\n    f for f in os.listdir(train_output)\n    if f.lower().endswith(\".jpg\")\n]\n\nprint(\"===== TRAIN VERIFICATION =====\")\nprint(\"Expected:\", len(train_df))\nprint(\"Found   :\", len(train_files))\nprint(\"Missing :\", len(train_df) - len(train_files))\n\nassert len(train_files) == len(train_df)\n\nprint(\"\\n✅ TRAIN preprocessing complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:42:52.282691Z","iopub.execute_input":"2026-10-08T19:42:52.283495Z","iopub.status.idle":"2026-10-08T19:42:52.297333Z","shell.execute_reply.started":"2026-10-08T19:42:52.283473Z","shell.execute_reply":"2026-10-08T19:42:52.296403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ntrain_output = os.path.join(\n    GRAHAM_384,\n    \"train\"\n)\n\ntrain_files = [\n    f for f in os.listdir(train_output)\n    if f.lower().endswith(\".jpg\")\n]\n\nprint(\"===== TRAIN OUTPUT DIAGNOSTIC =====\")\nprint(\"Expected images :\", len(train_df))\nprint(\"Current files   :\", len(train_files))\nprint(\"Missing images  :\", len(train_df) - len(train_files))\n\nprint(\"\\nFirst 10 processed files:\")\nfor f in sorted(train_files)[:10]:\n    print(\" \", f)\n\nprint(\"\\nLast 10 processed files:\")\nfor f in sorted(train_files)[-10:]:\n    print(\" \", f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:43:26.949976Z","iopub.execute_input":"2026-10-08T19:43:26.950277Z","iopub.status.idle":"2026-10-08T19:43:26.966872Z","shell.execute_reply.started":"2026-10-08T19:43:26.950253Z","shell.execute_reply":"2026-10-08T19:43:26.965669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"processed_names = {\n    os.path.splitext(f)[0]\n    for f in train_files\n}\n\nexpected_names = set(train_df[\"image\"])\n\nextra_files = processed_names - expected_names\nmissing_files = expected_names - processed_names\n\nprint(\"===== PROCESSED FILE CHECK =====\")\nprint(\"Expected train images :\", len(expected_names))\nprint(\"Processed files       :\", len(processed_names))\nprint(\"Missing files         :\", len(missing_files))\nprint(\"Unexpected files      :\", len(extra_files))\n\nif extra_files:\n    print(\"\\nUnexpected examples:\")\n    for name in sorted(extra_files)[:10]:\n        print(\" \", name)\n\nprint(\"\\nMissing examples:\")\nfor name in sorted(missing_files)[:10]:\n    print(\" \", name)\n\nassert len(extra_files) == 0\n\nprint(\"\\n✅ All existing processed images belong to the fixed TRAIN split.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:44:12.475427Z","iopub.execute_input":"2026-10-08T19:44:12.475686Z","iopub.status.idle":"2026-10-08T19:44:12.499735Z","shell.execute_reply.started":"2026-10-08T19:44:12.475666Z","shell.execute_reply":"2026-10-08T19:44:12.498965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Resuming TRAIN preprocessing...\")\nprint(\"Already processed:\", len(processed_names))\nprint(\"Remaining:\", len(missing_files))\n\ntrain_result = process_split(\n    dataframe=train_df,\n    split_name=\"train\",\n    zip_file=train_zip,\n    output_dir=GRAHAM_384\n)\n\nprint(\"\\n===== TRAIN RESULT =====\")\nprint(train_result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T19:48:28.208828Z","iopub.execute_input":"2026-10-08T19:48:28.209125Z","iopub.status.idle":"2026-10-08T20:31:20.46195Z","shell.execute_reply.started":"2026-10-08T19:48:28.209106Z","shell.execute_reply":"2026-10-08T20:31:20.460743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_output = os.path.join(\n    GRAHAM_384,\n    \"train\"\n)\n\ntrain_files = [\n    f for f in os.listdir(train_output)\n    if f.lower().endswith(\".jpg\")\n]\n\nprint(\"===== FINAL TRAIN CHECK =====\")\nprint(\"Expected :\", len(train_df))\nprint(\"Found    :\", len(train_files))\nprint(\"Missing  :\", len(train_df) - len(train_files))\n\nassert len(train_files) == len(train_df)\n\nprint(\"\\n✅ ALL 24,588 TRAIN IMAGES ARE READY.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T20:37:37.204082Z","iopub.execute_input":"2026-10-08T20:37:37.204644Z","iopub.status.idle":"2026-10-08T20:37:37.231305Z","shell.execute_reply.started":"2026-10-08T20:37:37.204615Z","shell.execute_reply":"2026-10-08T20:37:37.230694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting VALIDATION preprocessing...\")\n\nval_result = process_split(\n    dataframe=val_df,\n    split_name=\"validation\",\n    zip_file=train_zip,\n    output_dir=GRAHAM_384\n)\n\nprint(\"\\n===== VALIDATION RESULT =====\")\nprint(val_result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T20:38:07.31559Z","iopub.execute_input":"2026-10-08T20:38:07.316286Z","iopub.status.idle":"2026-10-08T20:53:45.054818Z","shell.execute_reply.started":"2026-10-08T20:38:07.316263Z","shell.execute_reply":"2026-10-08T20:53:45.054092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting TEST preprocessing...\")\n\ntest_result = process_split(\n    dataframe=test_df,\n    split_name=\"test\",\n    zip_file=train_zip,\n    output_dir=GRAHAM_384\n)\n\nprint(\"\\n===== TEST RESULT =====\")\nprint(test_result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T20:56:02.064549Z","iopub.execute_input":"2026-10-08T20:56:02.064871Z","iopub.status.idle":"2026-10-08T21:11:42.527861Z","shell.execute_reply.started":"2026-10-08T20:56:02.064851Z","shell.execute_reply":"2026-10-08T21:11:42.526798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom PIL import Image\n\nsplit_info = {\n    \"train\": train_df,\n    \"validation\": val_df,\n    \"test\": test_df\n}\n\nprint(\"===== FINAL DATASET VERIFICATION =====\")\n\ntotal_expected = 0\ntotal_found = 0\n\nfor split_name, df in split_info.items():\n\n    split_dir = os.path.join(\n        GRAHAM_384,\n        split_name\n    )\n\n    files = [\n        f for f in os.listdir(split_dir)\n        if f.lower().endswith(\".jpg\")\n    ]\n\n    expected = len(df)\n    found = len(files)\n\n    print(f\"\\n{split_name.upper()}\")\n    print(\"Expected:\", expected)\n    print(\"Found   :\", found)\n    print(\"Missing :\", expected - found)\n\n    assert found == expected\n\n    total_expected += expected\n    total_found += found\n\nprint(\"\\n===== TOTAL =====\")\nprint(\"Expected:\", total_expected)\nprint(\"Found   :\", total_found)\n\nassert total_found == total_expected\n\n# Check a few actual images\nfor split_name in split_info:\n\n    split_dir = os.path.join(\n        GRAHAM_384,\n        split_name\n    )\n\n    sample_file = next(\n        f for f in os.listdir(split_dir)\n        if f.lower().endswith(\".jpg\")\n    )\n\n    sample_path = os.path.join(\n        split_dir,\n        sample_file\n    )\n\n    with Image.open(sample_path) as img:\n        print(\n            f\"{split_name}:\",\n            img.size,\n            img.mode\n        )\n\nprint(\"\\n✅ ALL 35,126 PREPROCESSED IMAGES VERIFIED.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-08T21:12:42.476805Z","iopub.execute_input":"2026-10-08T21:12:42.478408Z","iopub.status.idle":"2026-10-08T21:12:42.533284Z","shell.execute_reply.started":"2026-10-08T21:12:42.478379Z","shell.execute_reply":"2026-10-08T21:12:42.532449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}