{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":33204,"databundleVersionId":3011186,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-22T06:21:56.791572Z","iopub.execute_input":"2025-10-22T06:21:56.79198Z","iopub.status.idle":"2025-10-22T06:22:22.948533Z","shell.execute_reply.started":"2025-10-22T06:21:56.791947Z","shell.execute_reply":"2025-10-22T06:22:22.947661Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nprint(torch.cuda.is_available(), torch.cuda.get_device_name(0))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T07:03:33.334367Z","iopub.execute_input":"2025-10-23T07:03:33.334609Z","iopub.status.idle":"2025-10-23T07:03:33.338829Z","shell.execute_reply.started":"2025-10-23T07:03:33.334592Z","shell.execute_reply":"2025-10-23T07:03:33.338039Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SETUP","metadata":{}},{"cell_type":"code","source":"# !pip install --quiet ultralytics==8.3.50 albumentations==1.4.7 pycocotools==2.0.8 \\\n#     opencv-python==4.10.0.84 matplotlib==3.9.2 numpy==1.26.4 pillow==10.4.0 torchmetrics==1.4.2\n\nimport os, random, math, shutil, time\nfrom pathlib import Path\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\n\nDATA_ROOT = Path(\"/kaggle/input/find-person-in-the-dark\")\n\nIM_DIR_TRAIN  = DATA_ROOT / \"train\" / \"train\" / \"train_images\"\nANN_DIR_TRAIN = DATA_ROOT / \"train\" / \"train\" / \"train_annotations\"\nIM_DIR_TEST   = DATA_ROOT / \"test\"  / \"test\"  / \"test_images\"\n\nOUT_DIR = Path(\"./outputs\"); OUT_DIR.mkdir(exist_ok=True, parents=True)\nWORK_ROOT = Path(\"/kaggle/working/darkdata\"); WORK_ROOT.mkdir(exist_ok=True, parents=True)\n\nYOLO_TRAIN_IMG = WORK_ROOT / \"train\" / \"images\"\nYOLO_TRAIN_LBL = WORK_ROOT / \"train\" / \"labels\"\nYOLO_VAL_IMG   = WORK_ROOT / \"val\"   / \"images\"\nYOLO_VAL_LBL   = WORK_ROOT / \"val\"   / \"labels\"\nYOLO_TEST_IMG  = WORK_ROOT / \"test\"  / \"images\"\nfor p in [YOLO_TRAIN_IMG, YOLO_TRAIN_LBL, YOLO_VAL_IMG, YOLO_VAL_LBL, YOLO_TEST_IMG]:\n    p.mkdir(parents=True, exist_ok=True)\n\nCLASSES = [\"person\"]\nNUM_CLASSES = 1\n\nUSE_PRETRAINED = False\n#MODEL_NAME = \"yolov8n\"\nMODEL_NAME=\"yolo11n\"\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED)\nprint(\"Train image folder:\", IM_DIR_TRAIN)\nprint(\"Annotation folder :\", ANN_DIR_TRAIN)\nprint(\"Test image folder :\", IM_DIR_TEST)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:21:06.357903Z","iopub.execute_input":"2025-10-23T05:21:06.358449Z","iopub.status.idle":"2025-10-23T05:21:06.624695Z","shell.execute_reply.started":"2025-10-23T05:21:06.358426Z","shell.execute_reply":"2025-10-23T05:21:06.624038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk(\"/kaggle/input\"):\n    for f in filenames[:3]:\n        print(os.path.join(dirname, f))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T03:44:06.552955Z","iopub.execute_input":"2025-10-23T03:44:06.553695Z","iopub.status.idle":"2025-10-23T03:45:02.92645Z","shell.execute_reply.started":"2025-10-23T03:44:06.553662Z","shell.execute_reply":"2025-10-23T03:45:02.925624Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Exporative Data Analysis ","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport random, math\nimport cv2\nimport matplotlib.pyplot as plt\n\ndef list_images(root: Path):\n    exts = ('.jpg', '.jpeg', '.png', '.bmp')\n    if root is None or not root.exists():\n        return []\n    return [p for p in root.rglob('*') if p.suffix.lower() in exts]\n\ntrain_imgs = list_images(IM_DIR_TRAIN)          # DATA_ROOT/train/train_images\nval_imgs   = []                                  # val split\ntest_imgs  = list_images(IM_DIR_TEST)           # DATA_ROOT/test/test_images\n\nprint(f\"Train images: {len(train_imgs)}, Val images: {len(val_imgs)}, Test images: {len(test_imgs)}\")\n\n# showing samples\ndef show_samples(paths, n=6, title='Samples'):\n    if not paths:\n        print(\"No images found to display.\")\n        return\n    n = min(n, len(paths))\n    sel = random.sample(paths, n)\n    cols = min(3, n)\n    rows = math.ceil(n / cols)\n    plt.figure(figsize=(4*cols, 3.5*rows))\n    for i, p in enumerate(sel, 1):\n        img = cv2.imread(str(p))\n        if img is None:\n            continue\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        plt.subplot(rows, cols, i)\n        plt.imshow(img)\n        plt.axis('off')\n        plt.title(p.name)\n    plt.suptitle(title)\n    plt.show()\n\nshow_samples(train_imgs, n=6, title='Train Samples (Random)')\n# show_samples(test_imgs,  n=6, title='Test Samples (Random)')  # uncomment for test migs view\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:21:09.025839Z","iopub.execute_input":"2025-10-23T05:21:09.026147Z","iopub.status.idle":"2025-10-23T05:21:54.314791Z","shell.execute_reply.started":"2025-10-23T05:21:09.026126Z","shell.execute_reply":"2025-10-23T05:21:54.313863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#brightness and contrast\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport random\nimport numpy as np\n\ndef compute_brightness_contrast(img_bgr):\n    \"\"\"Return mean (brightness) and std (contrast) of the grayscale version.\"\"\"\n    gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)\n    mean = float(gray.mean())\n    std  = float(gray.std())\n    return mean, std\n\ndef plot_brightness_distribution(paths, k=500):\n    \"\"\"Sample up to k images and plot histograms of brightness and contrast.\"\"\"\n    if not paths:\n        print(\"No paths for distribution.\")\n        return [], []\n    subset = random.sample(paths, min(k, len(paths)))\n    br, co = [], []\n    for p in subset:\n        img = cv2.imread(str(p))\n        if img is None:\n            continue\n        m, s = compute_brightness_contrast(img)\n        br.append(m)\n        co.append(s)\n    plt.figure(figsize=(6, 4))\n    plt.hist(br, bins=30, color='steelblue', edgecolor='black')\n    plt.title('Brightness (gray mean)')\n    plt.xlabel('mean intensity (0–255)')\n    plt.ylabel('count')\n    plt.show()\n\n    plt.figure(figsize=(6, 4))\n    plt.hist(co, bins=30, color='orange', edgecolor='black')\n    plt.title('Contrast (gray std)')\n    plt.xlabel('standard deviation')\n    plt.ylabel('count')\n    plt.show()\n    return br, co\n\n#running on a random subset of the training set\n_ = plot_brightness_distribution(train_imgs, k=400)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:21:58.683308Z","iopub.execute_input":"2025-10-23T05:21:58.68403Z","iopub.status.idle":"2025-10-23T05:22:10.462652Z","shell.execute_reply.started":"2025-10-23T05:21:58.684006Z","shell.execute_reply":"2025-10-23T05:22:10.462039Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Based on the results we may conclude that the dataset has:\n\na) Low light bias. The brightness distribution is skewed toward darker values, and the majority of images have intensities below 40/255 (max 100).\nMost scenes are captured under poor illumination. Such distribution poses a challenge for feature extraction, since shadowed regions may obscure human silhouettes.\n\nb) High contrast variability - the contrast histogram is widely spread, ranging from 10 to almost 70, indicating differing lighting conditions.\nSome images are nearly flat (low contrast), while others exhibit extreme differences between bright and dark areas. This variation complicates model generalization because exposure patterns vary drastically between samples.\n\nImplication: These findings highlight the importance of illumination-based augmentations, so we will try to use CLAHE and Gamma correction.","metadata":{}},{"cell_type":"code","source":"def clahe_enhance(img_bgr, clip_limit=2.0, tile_grid_size=(8, 8)):\n    \"\"\"Apply CLAHE (adaptive histogram equalization) to the L channel in LAB space.\"\"\"\n    lab = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2LAB)\n    l, a, b = cv2.split(lab)\n    clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=tile_grid_size)\n    cl = clahe.apply(l)\n    merged = cv2.merge([cl, a, b])\n    return cv2.cvtColor(merged, cv2.COLOR_LAB2BGR)\n\ndef gamma_correction(img_bgr, gamma=1.5):\n    \"\"\"Apply gamma correction (γ > 1 brightens, γ < 1 darkens).\"\"\"\n    invG = 1.0 / gamma\n    table = np.array([(i / 255.0) ** invG * 255 for i in np.arange(256)]).astype(\"uint8\")\n    return cv2.LUT(img_bgr, table)\n\n# Pick one random image to preview\nif train_imgs:\n    p = random.choice(train_imgs)\n    base = cv2.imread(str(p))\n    if base is not None:\n        c_img = clahe_enhance(base, 2.0, (8, 8))\n        g_img = gamma_correction(base, 1.8)\n        plt.figure(figsize=(12, 4))\n        for i, (im, t) in enumerate([(base, 'Original'), (c_img, 'CLAHE'), (g_img, 'Gamma 1.8')]):\n            plt.subplot(1, 3, i + 1)\n            plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))\n            plt.axis('off')\n            plt.title(t)\n        plt.suptitle('Low-light enhancement preview (train-time aug only, not test-time)')\n        plt.show()\n    else:\n        print(f\"Could not load image: {p}\")\nelse:\n    print(\"No train images found to visualize.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:22:16.914176Z","iopub.execute_input":"2025-10-23T05:22:16.914443Z","iopub.status.idle":"2025-10-23T05:22:17.815523Z","shell.execute_reply.started":"2025-10-23T05:22:16.914424Z","shell.execute_reply":"2025-10-23T05:22:17.814766Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model training","metadata":{}},{"cell_type":"markdown","source":"Model used: yolov11.\n\nYOLOv11 is a current SOTA YOLO-family model that combines reasonable size, good inference speed, reliable performance, and is especially able with low-light detection tasks. The model utilizes the anchor-free detection head, enabling it to generalize better across varying scales and aspect ratios of humans without requiring manual anchor tuning, helping with diverse silhouettes. Moreover, Ultralytics’ API supports direct model.train(), model.val(), and model.predict() pipelines with COCO-style metrics and automatic export, simplifying experimentation and evaluation steps.","metadata":{}},{"cell_type":"code","source":"#image augmentation\n#these augmentations will be applied during training to make the model robust to low-light variations.\n\nfrom albumentations import (\n    Compose, OneOf, BboxParams, HorizontalFlip, RandomBrightnessContrast,\n    GaussianBlur, MotionBlur, GaussNoise, HueSaturationValue, CLAHE,\n    RandomGamma\n)\n\ndef make_train_transforms(img_size=640):\n    return Compose([\n        HorizontalFlip(p=0.5),#geometry\n        OneOf([ #illumnation\n            #random brightness and contrast shift\n            RandomBrightnessContrast(brightness_limit=0.5, contrast_limit=0.4, p=0.9),\n            #brightening dark regions\n            CLAHE(clip_limit=(1.0, 3.0), p=0.6),\n            #gamma correction\n            RandomGamma(gamma_limit=(120, 220), p=0.8)\n        ], p=0.9),\n\n        OneOf([\n            GaussianBlur(blur_limit=3, p=0.3),#mild gaussian blur\n            MotionBlur(blur_limit=5, p=0.3),  #motion streaks\n            GaussNoise(var_limit=(5.0, 30.0), p=0.4) #sensor noise\n        ], p=0.6),\n\n        HueSaturationValue( #color variation\n            hue_shift_limit=0,#leave hue stable (night scenes)\n            sat_shift_limit=10,#slight saturation alteration\n            val_shift_limit=10,#brightness tweak\n            p=0.3\n        ),\n    ],\n    #YOLO expects normalized format for boxes\n    bbox_params=BboxParams(format='yolo', label_fields=['class_labels'])\n    )\n\nprint(\"Augmentation pipeline is ready.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:22:20.868134Z","iopub.execute_input":"2025-10-23T05:22:20.868848Z","iopub.status.idle":"2025-10-23T05:22:22.443167Z","shell.execute_reply.started":"2025-10-23T05:22:20.868824Z","shell.execute_reply":"2025-10-23T05:22:22.442354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#YOLO-format dataset yaml writer \nfrom pathlib import Path\n\ndef write_data_yaml(yolo_root: Path, classes, out_yaml='dark_person.yaml'):\n    \"\"\"\n    Write the dataset YAML file Ultralytics expects.\n    - yolo_root: dataset root (e.g. /kaggle/working/darkdata)\n    - classes: list of class names\n    \"\"\"\n    yaml_text = f\"\"\"# auto-generated\npath: {yolo_root.as_posix()}\ntrain: train/images\nval: val/images\ntest: test/images\nnames: {classes}\n\"\"\"\n    outp = Path(out_yaml)\n    outp.write_text(yaml_text.strip())\n    print(f\"Wrote dataset YAML: {outp.resolve()}\")\n    print(outp.read_text())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:22:24.071118Z","iopub.execute_input":"2025-10-23T05:22:24.071544Z","iopub.status.idle":"2025-10-23T05:22:24.076355Z","shell.execute_reply.started":"2025-10-23T05:22:24.071521Z","shell.execute_reply":"2025-10-23T05:22:24.075727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:22:28.282254Z","iopub.execute_input":"2025-10-23T05:22:28.282865Z","iopub.status.idle":"2025-10-23T05:23:46.131139Z","shell.execute_reply.started":"2025-10-23T05:22:28.282833Z","shell.execute_reply":"2025-10-23T05:23:46.130354Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_NAME=\"yolo11n\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:23:58.959082Z","iopub.execute_input":"2025-10-23T05:23:58.959374Z","iopub.status.idle":"2025-10-23T05:23:58.963691Z","shell.execute_reply.started":"2025-10-23T05:23:58.959353Z","shell.execute_reply":"2025-10-23T05:23:58.963124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#converting Pascal VOC → YOLO format and create train/val splits\n\nimport xml.etree.ElementTree as ET\nfrom sklearn.model_selection import train_test_split\nimport shutil\nfrom tqdm import tqdm\n\ndef voc_to_yolo(xml_path, class_list):\n    tree = ET.parse(xml_path)\n    root = tree.getroot()\n\n    size = root.find(\"size\")\n    w = int(size.find(\"width\").text)\n    h = int(size.find(\"height\").text)\n\n    labels = []\n    for obj in root.findall(\"object\"):\n        cls = obj.find(\"name\").text.strip().lower()\n        if cls not in class_list:\n            continue\n        cls_id = class_list.index(cls)\n        box = obj.find(\"bndbox\")\n        x1 = float(box.find(\"xmin\").text)\n        y1 = float(box.find(\"ymin\").text)\n        x2 = float(box.find(\"xmax\").text)\n        y2 = float(box.find(\"ymax\").text)\n\n        # Convert to YOLO normalized format\n        x_center = (x1 + x2) / 2.0 / w\n        y_center = (y1 + y2) / 2.0 / h\n        bw = (x2 - x1) / w\n        bh = (y2 - y1) / h\n        labels.append(f\"{cls_id} {x_center:.6f} {y_center:.6f} {bw:.6f} {bh:.6f}\")\n    return labels\n#gatheirng img paths\nimg_paths = sorted([p for p in IM_DIR_TRAIN.glob(\"*.jpg\")])\nprint(f\"Found {len(img_paths)} training images.\")\n\n# train/val split \ntrain_imgs, val_imgs = train_test_split(img_paths, test_size=0.15, random_state=SEED)\n\ndef copy_and_convert(img_list, dest_img_dir, dest_lbl_dir):\n    dest_img_dir.mkdir(parents=True, exist_ok=True)\n    dest_lbl_dir.mkdir(parents=True, exist_ok=True)\n    for img_path in tqdm(img_list, desc=f\"Converting to {dest_img_dir.parent.name}\"):\n        xml_path = ANN_DIR_TRAIN / (img_path.stem + \".xml\")\n        if not xml_path.exists():\n            continue\n        labels = voc_to_yolo(xml_path, CLASSES)\n        if not labels:\n            continue\n        # copy image\n        shutil.copy2(img_path, dest_img_dir / img_path.name)\n        # write label\n        with open(dest_lbl_dir / f\"{img_path.stem}.txt\", \"w\") as f:\n            f.write(\"\\n\".join(labels))\n\ncopy_and_convert(train_imgs, YOLO_TRAIN_IMG, YOLO_TRAIN_LBL)\ncopy_and_convert(val_imgs,   YOLO_VAL_IMG,   YOLO_VAL_LBL)\n\n# -copying test images\nfor p in tqdm(IM_DIR_TEST.glob(\"*.jpg\"), desc=\"Copying test images\"):\n    shutil.copy2(p, YOLO_TEST_IMG / p.name)\n\nprint(\"Conversion complete.\")\nprint(\"Train images:\", len(list(YOLO_TRAIN_IMG.glob('*.jpg'))))\nprint(\"Val images:\", len(list(YOLO_VAL_IMG.glob('*.jpg'))))\nprint(\"Test images:\", len(list(YOLO_TEST_IMG.glob('*.jpg'))))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:27:05.188574Z","iopub.execute_input":"2025-10-23T05:27:05.189049Z","iopub.status.idle":"2025-10-23T05:27:50.763834Z","shell.execute_reply.started":"2025-10-23T05:27:05.189025Z","shell.execute_reply":"2025-10-23T05:27:50.763012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"write_data_yaml(WORK_ROOT, CLASSES, out_yaml='dark_person.yaml')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:31:43.708864Z","iopub.execute_input":"2025-10-23T05:31:43.709398Z","iopub.status.idle":"2025-10-23T05:31:43.71399Z","shell.execute_reply.started":"2025-10-23T05:31:43.709376Z","shell.execute_reply":"2025-10-23T05:31:43.71317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!#nvidia-smi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T03:54:25.644234Z","iopub.execute_input":"2025-10-23T03:54:25.64498Z","iopub.status.idle":"2025-10-23T03:54:25.884463Z","shell.execute_reply.started":"2025-10-23T03:54:25.644948Z","shell.execute_reply":"2025-10-23T03:54:25.883661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Downgrade NumPy to a 1.x build (safe with Ultralytics and matplotlib)\n#!pip install --quiet \"numpy<2.0\" --force-reinstall","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:31:49.00942Z","iopub.execute_input":"2025-10-23T05:31:49.010042Z","iopub.status.idle":"2025-10-23T05:31:55.167851Z","shell.execute_reply.started":"2025-10-23T05:31:49.010018Z","shell.execute_reply":"2025-10-23T05:31:55.166929Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Training: \n\nAll annotations were converted to YOLO text format (.txt) using normalized bounding boxes. The dataset was defined via a dark_person.yaml file specifying paths and the single class label (person).\n\nConsidering the substantian variability in illumination and visibility, augmentation was essential to assist the model in generalization. Following techniques were employed: \n\nPhotometric (lighting conditions and color shifts), blur and noise (camera and nighttime noise), geometric (spatial diversity), regularization (robustness, small objects detection).\n\nYolo11n with batch size 16 was used to fit in the GPU restrictions, imgsz 512 and 30 epochs for the training speed, lr=0.01, optimized=\"Adam\" as a standard. Standard YOLO composite loss was used. hsv_h =0.02, hsv_s=0.5, hsv_v=0.4 were chosen to accomodate the model to the low light object detection task\n","metadata":{}},{"cell_type":"code","source":"#model selection & training\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n\nfrom ultralytics import YOLO\nmodel = YOLO(\"yolo11n.yaml\")\n\n# internally calls same trainer as CLI with DDP\nmodel.train(\n    data=\"dark_person.yaml\",\n    epochs=30,\n    imgsz=512,\n    batch=16,\n    device=\"0,1\",\n    project=\"./outputs/train\",\n    name=\"yolo11n_ddp_py\",\n    lr0 = 0.01,\n    optimizer=\"Adam\",\n    hsv_h=0.02, \n    hsv_s=0.5, \n    hsv_v=0.4, \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T05:43:05.027009Z","iopub.execute_input":"2025-10-23T05:43:05.027287Z","iopub.status.idle":"2025-10-23T06:40:22.196521Z","shell.execute_reply.started":"2025-10-23T05:43:05.027269Z","shell.execute_reply":"2025-10-23T06:40:22.195861Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n\nbest_path = \"./outputs/train/yolo11n_ddp_py2/weights/best.pt\"\nmodel = YOLO(best_path)\n\nval_results = model.val(\n    data=\"dark_person.yaml\",\n    split=\"val\",\n    conf=0.25,\n    save_json=True,\n    save_txt=False,\n    save_hybrid=False,\n    save_dir=\"./outputs/coco_eval\",\n    device=\"0,1\",\n    verbose=True,\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T07:26:35.734259Z","iopub.execute_input":"2025-10-23T07:26:35.73454Z","iopub.status.idle":"2025-10-23T07:26:52.684153Z","shell.execute_reply.started":"2025-10-23T07:26:35.734519Z","shell.execute_reply":"2025-10-23T07:26:52.683437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nimport os\nfrom pathlib import Path\n\nmodel = YOLO(\"./outputs/train/yolo11n_ddp_py/weights/best.pt\")\n\ntest_dir = Path(\"/kaggle/working/darkdata/test/images\")\n\nresults = model.predict(\n    source=str(test_dir),\n    imgsz=512,\n    conf=0.25,\n    device=\"0,1\",\n    save=True,           # saves visualization images\n    save_txt=True,       # saves YOLO txt predictions\n    save_conf=True,      # include confidences\n    project=\"./outputs/test_inference\",\n    name=\"yolo11n_adam_test\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T06:55:44.638283Z","iopub.execute_input":"2025-10-23T06:55:44.638852Z","iopub.status.idle":"2025-10-23T06:57:28.023904Z","shell.execute_reply.started":"2025-10-23T06:55:44.638824Z","shell.execute_reply":"2025-10-23T06:57:28.023304Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nfrom pathlib import Path\nimport pandas as pd\n\nmodel = YOLO(\"./outputs/train/yolo11n_ddp_py2/weights/best.pt\")\ntest_dir = Path(\"/kaggle/working/darkdata/test/images\")\n\n#running predictions for all test images at once\nresults = model.predict(\n    source=str(test_dir),\n    imgsz=512,\n    conf=0.25,\n    device=\"0,1\",\n    save=False,\n    stream=False,  # ensures results list, not generator\n    verbose=False\n)\n\n# building the submission\nrows = []\nfor r in results:\n    image_id = Path(r.path).name\n    parts = []\n    for b in r.boxes:\n        x1, y1, x2, y2 = b.xyxy[0].cpu().numpy().tolist()\n        w, h = x2 - x1, y2 - y1\n        score = float(b.conf[0].cpu().item())\n        parts += [f\"{score:.6f}\", f\"{x1:.1f}\", f\"{y1:.1f}\", f\"{w:.1f}\", f\"{h:.1f}\"]\n    pred_str = \" \".join(parts)\n    rows.append({\"image_id\": image_id, \"PredictionString\": pred_str})\n\nsub = pd.DataFrame(rows)\nout_path = Path(\"./outputs/submission.csv\")\nout_path.parent.mkdir(exist_ok=True, parents=True)\nsub.to_csv(out_path, index=False)\nprint(f\"Saved {len(sub)} predictions to {out_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T07:24:36.079144Z","iopub.execute_input":"2025-10-23T07:24:36.079846Z","iopub.status.idle":"2025-10-23T07:25:35.000657Z","shell.execute_reply.started":"2025-10-23T07:24:36.079821Z","shell.execute_reply":"2025-10-23T07:25:34.999941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Validation Results Summary:\")\nprint(f\"mAP50: {val_results.box.map50:.3f}, mAP75: {val_results.box.map75:.3f}, mAP50-95: {val_results.box.map:.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T08:05:56.372448Z","iopub.execute_input":"2025-10-23T08:05:56.373167Z","iopub.status.idle":"2025-10-23T08:05:56.377512Z","shell.execute_reply.started":"2025-10-23T08:05:56.373144Z","shell.execute_reply":"2025-10-23T08:05:56.376729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### The results are mAP50: 0.872, mAP75: 0.505, mAP50-95: 0.497. The results are of fair quality, presenting good results with the GPU and time restrictions\n\n#### Let's see how the model detects objects and compare with the ground truth boxes","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\nimport cv2, matplotlib.pyplot as plt\nfrom pathlib import Path\nimport random\n\nmodel = YOLO(\"./outputs/train/yolo11n_ddp_py2/weights/best.pt\")\n\ntest_dir = Path(\"/kaggle/working/darkdata/test/images\")\nsample_imgs = random.sample(list(test_dir.glob(\"*.jpg\")), 6)\n\nplt.figure(figsize=(15, 10))\nfor i, img_path in enumerate(sample_imgs, 1):\n    r = model.predict(source=str(img_path), imgsz=512, conf=0.25, verbose=False)[0]\n    img = cv2.imread(str(img_path))\n    for box in r.boxes:\n        x1, y1, x2, y2 = map(int, box.xyxy[0])\n        cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2)\n    plt.subplot(2, 3, i)\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n    plt.title(Path(img_path).name)\n    plt.axis(\"off\")\n\nplt.suptitle(\"Predicted Bounding Boxes (Green)\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T07:21:30.530617Z","iopub.execute_input":"2025-10-23T07:21:30.53128Z","iopub.status.idle":"2025-10-23T07:21:32.613444Z","shell.execute_reply.started":"2025-10-23T07:21:30.531252Z","shell.execute_reply":"2025-10-23T07:21:32.612485Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's build a JSON file with the GT boxes","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nfrom pathlib import Path\nfrom PIL import Image\n\n# Paths\nyolo_labels = Path(\"/kaggle/working/darkdata/val/labels\")\nimages_dir = Path(\"/kaggle/working/darkdata/val/images\")\noutput_json = Path(\"/kaggle/working/outputs/val_gt_valonly.json\")\n\ncategories = [{\"id\": 1, \"name\": \"person\"}]\n\nimages = []\nannotations = []\nann_id = 1\n\nfor i, img_file in enumerate(sorted(images_dir.glob(\"*.jpg\"))):\n    img = Image.open(img_file)\n    w, h = img.size\n    img_id = i + 1\n    images.append({\n        \"id\": img_id,\n        \"file_name\": img_file.name,\n        \"width\": w,\n        \"height\": h\n    })\n    label_file = yolo_labels / (img_file.stem + \".txt\")\n    if not label_file.exists():\n        continue\n\n    with open(label_file) as f:\n        for line in f:\n            parts = line.strip().split()\n            if len(parts) != 5:\n                continue\n            cls, x_center, y_center, bw, bh = map(float, parts)\n            x = (x_center - bw / 2) * w\n            y = (y_center - bh / 2) * h\n            bw *= w\n            bh *= h\n            annotations.append({\n                \"id\": ann_id,\n                \"image_id\": img_id,\n                \"category_id\": int(cls) + 1,\n                \"bbox\": [x, y, bw, bh],\n                \"area\": bw * bh,\n                \"iscrowd\": 0\n            })\n            ann_id += 1\n\ncoco_json = {\n    \"images\": images,\n    \"annotations\": annotations,\n    \"categories\": categories,\n}\n\noutput_json.parent.mkdir(parents=True, exist_ok=True)\nwith open(output_json, \"w\") as f:\n    json.dump(coco_json, f, indent=2)\n\nprint(f\"Saved COCO-format JSON to {output_json}\")\nprint(f\"Images: {len(images)}, Annotations: {len(annotations)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T07:32:39.875827Z","iopub.execute_input":"2025-10-23T07:32:39.876455Z","iopub.status.idle":"2025-10-23T07:32:40.715647Z","shell.execute_reply.started":"2025-10-23T07:32:39.876432Z","shell.execute_reply":"2025-10-23T07:32:40.714852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def draw_gt_pred(img_bgr, gt_boxes, pred_boxes, gt_color=(0,0,255), pred_color=(0,255,0)):\n    \"\"\"Draw GT (red) and Predicted (green) boxes on an image.\"\"\"\n    img = img_bgr.copy()\n    # Ground truth boxes\n    for (x1, y1, x2, y2) in gt_boxes:\n        cv2.rectangle(img, (int(x1), int(y1)), (int(x2), int(y2)), gt_color, 2)\n    # Predicted boxes\n    for (x1, y1, x2, y2) in pred_boxes:\n        cv2.rectangle(img, (int(x1), int(y1)), (int(x2), int(y2)), pred_color, 2)\n    return img\n\ndef visualize_gt_pred(model, coco_json, img_dir, n=6, conf=0.25):\n    \"\"\"Visualize random validation images with GT and predictions.\"\"\"\n    with open(coco_json) as f:\n        coco = json.load(f)\n\n    ann_by_img = {}\n    for ann in coco[\"annotations\"]:\n        x, y, w, h = ann[\"bbox\"]\n        ann_by_img.setdefault(ann[\"image_id\"], []).append([x, y, x+w, y+h])\n\n    id2file = {img[\"id\"]: img[\"file_name\"] for img in coco[\"images\"]}\n    selected = random.sample(list(id2file.keys()), min(n, len(id2file)))\n\n    cols = min(3, n)\n    rows = int(np.ceil(n / cols))\n    plt.figure(figsize=(6 * cols, 6 * rows))\n\n    for i, img_id in enumerate(selected, 1):\n        img_path = Path(img_dir) / id2file[img_id]\n        img = cv2.imread(str(img_path))\n        if img is None:\n            continue\n\n        gt_boxes = ann_by_img.get(img_id, [])\n        r = model.predict(source=str(img_path), imgsz=512, conf=conf, verbose=False)[0]\n\n        pred_boxes = []\n        if r.boxes is not None:\n            for b in r.boxes:\n                x1, y1, x2, y2 = b.xyxy[0].cpu().numpy().tolist()\n                pred_boxes.append([x1, y1, x2, y2])\n\n        vis = draw_gt_pred(img, gt_boxes, pred_boxes)\n        plt.subplot(rows, cols, i)\n        plt.imshow(cv2.cvtColor(vis, cv2.COLOR_BGR2RGB))\n        plt.axis(\"off\")\n        plt.title(Path(img_path).name)\n    plt.suptitle(\"Ground Truth vs Predictions\", fontsize=18)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T08:09:50.994373Z","iopub.execute_input":"2025-10-23T08:09:50.995016Z","iopub.status.idle":"2025-10-23T08:09:51.005109Z","shell.execute_reply.started":"2025-10-23T08:09:50.994992Z","shell.execute_reply":"2025-10-23T08:09:51.004216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nmodel = YOLO(\"./outputs/train/yolo11n_ddp_py/weights/best.pt\")\n\nvisualize_gt_pred(\n    model,\n    coco_json=\"/kaggle/working/outputs/val_gt_valonly.json\",  # your validation GT JSON\n    img_dir=\"/kaggle/working/darkdata/val/images\",\n    n=6,\n    conf=0.25\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T08:09:55.790863Z","iopub.execute_input":"2025-10-23T08:09:55.791181Z","iopub.status.idle":"2025-10-23T08:09:58.322833Z","shell.execute_reply.started":"2025-10-23T08:09:55.791158Z","shell.execute_reply":"2025-10-23T08:09:58.322047Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Just as we have seen with precision and recall, the model tends to spawn False Negatives, failing to detect some of the silhouttes on the images. We can attribute it to the lack of the epochs (since there is a quality improvement trend up to the 30 epochs), and to the problems with the detection of people covered by other objects. I presume rotational augmentations could have been used during the training step to make the model more sensitive to these cases.","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\nfrom pathlib import Path\nimport os\n\nmodel = YOLO(\"./outputs/train/yolo11n_ddp_py2/weights/best.pt\")\ntest_dir = Path(\"/kaggle/working/darkdata/test/images\")\noutput_dir = Path(\"/kaggle/working/evaluate/detections\")\noutput_dir.mkdir(parents=True, exist_ok=True)\n\n# run inference\nresults = model.predict(\n    source=str(test_dir),\n    imgsz=512,\n    conf=0.25,\n    device=\"0,1\",\n    save=False,\n    stream=False,\n    verbose=False\n)\n\n# export predictions into text files\nfor r in results:\n    image_id = Path(r.path).stem\n    lines = []\n    for b in r.boxes:\n        cls = model.names[int(b.cls[0])]\n        conf = float(b.conf[0])\n        x1, y1, x2, y2 = b.xyxy[0].cpu().numpy().tolist()\n        w, h = x2 - x1, y2 - y1\n        lines.append(f\"{cls} {conf:.4f} {x1:.1f} {y1:.1f} {w:.1f} {h:.1f}\")\n    \n    with open(output_dir / f\"{image_id}.txt\", \"w\") as f:\n        f.write(\"\\n\".join(lines))\n\nprint(f\"Saved {len(results)} detection files to {output_dir}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T08:17:45.676662Z","iopub.execute_input":"2025-10-23T08:17:45.677082Z","iopub.status.idle":"2025-10-23T08:18:45.753849Z","shell.execute_reply.started":"2025-10-23T08:17:45.67706Z","shell.execute_reply":"2025-10-23T08:18:45.753097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nfrom pathlib import Path\n\nbase = Path(\"/kaggle/working/evaluate\")\nsrc = Path(\"/kaggle/input/find-person-in-the-dark/evaluate\")\nbase.mkdir(parents=True, exist_ok=True)\n\nfor item in src.iterdir():\n    if item.name == \"evaluate\" and item.is_dir():\n        print(f\"Unpacking nested source directory: {item.name}\")\n        \n        for sub_item in item.iterdir():\n            dest_sub = base / sub_item.name\n            if dest_sub.exists():\n                continue\n            if sub_item.is_dir():\n                shutil.copytree(sub_item, dest_sub)\n            else:\n                shutil.copy2(sub_item, dest_sub)\n        continue \n    \n    dest = base / item.name\n    \n    if dest.exists():\n        continue \n    \n    if item.is_dir():\n        shutil.copytree(item, dest)\n    else:\n        shutil.copy2(item, dest) \n\nprint(\"Evaluator files successfully copied and unpacked into /kaggle/working/evaluate.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T08:55:21.924405Z","iopub.execute_input":"2025-10-23T08:55:21.925034Z","iopub.status.idle":"2025-10-23T08:55:26.654746Z","shell.execute_reply.started":"2025-10-23T08:55:21.925009Z","shell.execute_reply":"2025-10-23T08:55:26.654043Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SUBMISSION","metadata":{}},{"cell_type":"code","source":"!cd /kaggle/working/evaluate && echo \"Y\" | python pascalvoc.py --threshold 0.5\n!cd /kaggle/working/evaluate && echo \"Y\" | python pascalvoc.py --threshold 0.75","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T09:02:42.700069Z","iopub.execute_input":"2025-10-23T09:02:42.700417Z","iopub.status.idle":"2025-10-23T09:02:46.907787Z","shell.execute_reply.started":"2025-10-23T09:02:42.700392Z","shell.execute_reply":"2025-10-23T09:02:46.907043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\n\nap50 = 0.7930\nap75 = 0.4203\n\nsubmission = pd.DataFrame({\n    \"ID\": [\"AP@0.5\", \"AP@0.75\"],\n    \"value\": [round(ap50, 4), round(ap75, 4)]\n})\n\nout_path = Path(\"/kaggle/working/submission.csv\")\nsubmission.to_csv(out_path, index=False)\nprint(f\"Submission file saved to: {out_path.resolve()}\")\n\ndisplay(submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-23T09:08:34.176732Z","iopub.execute_input":"2025-10-23T09:08:34.177079Z","iopub.status.idle":"2025-10-23T09:08:34.214569Z","shell.execute_reply.started":"2025-10-23T09:08:34.177054Z","shell.execute_reply":"2025-10-23T09:08:34.213936Z"}},"outputs":[],"execution_count":null}]}