{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":129750,"databundleVersionId":15563344}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ultralytics -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:06.128427Z","iopub.execute_input":"2026-02-20T03:40:06.128760Z","iopub.status.idle":"2026-02-20T03:40:10.579960Z","shell.execute_reply.started":"2026-02-20T03:40:06.128736Z","shell.execute_reply":"2026-02-20T03:40:10.579237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#cell 0\n# Import libraries\nimport os\nimport shutil\nimport pandas as pd\nfrom pathlib import Path\nfrom ultralytics import YOLO\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport os\nimport random\nimport numpy as np\n\nSEED = 42\n\n# (1) Python + hash\nos.environ[\"PYTHONHASHSEED\"] = str(SEED)\n\n# (2) cuBLAS deterministic (must be set BEFORE torch import in ideal case)\n# In notebooks it still helps; if you want max strictness, put this in the first cell of the notebook.\nos.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\nimport torch\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\n\n# (3) Determinism flags\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\n# (4) PyTorch \"deterministic algorithms\" (can throw if an op is nondeterministic)\ntry:\n    torch.use_deterministic_algorithms(True)\nexcept Exception as e:\n    print(\"Deterministic algos warning:\", e)\n\nprint(\"✅ Seeds set:\", SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:10.581514Z","iopub.execute_input":"2026-02-20T03:40:10.581772Z","iopub.status.idle":"2026-02-20T03:40:18.378861Z","shell.execute_reply.started":"2026-02-20T03:40:10.581743Z","shell.execute_reply":"2026-02-20T03:40:18.378103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# data prerpation ","metadata":{}},{"cell_type":"markdown","source":"## drop duplicate images ","metadata":{}},{"cell_type":"code","source":"import os, hashlib, math\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# --- CONFIG ---\nROOT = \"/kaggle/input/dal-shemagh-detection-challenge\"\nIMG_DIR, LBL_DIR = os.path.join(ROOT, \"images/train\"), os.path.join(ROOT, \"labels/train\")\nKEEP_STRATEGY = \"largest_labels\" # \"largest_labels\" or \"smallest_name\"\nFORCE_KEEP, FORCE_DROP = {\"638.jpg\"}, {\"440.jpg\"}\nPLOT, MAX_GROUPS, COLS = True, 10, 3\n\n# --- HELPERS ---\ndef get_md5(path):\n    with open(path, \"rb\") as f: return hashlib.md5(f.read()).hexdigest()\n\ndef get_yolo(fn):\n    p = os.path.join(LBL_DIR, fn.rsplit('.', 1)[0] + \".txt\")\n    if not os.path.exists(p): return []\n    try: return [list(map(float, l.strip().split())) for l in open(p)]\n    except: return []\n\n# --- 1. LOAD & HASH ---\ndf = pd.read_csv(os.path.join(ROOT, \"train_labels.csv\")).drop_duplicates(\"filename\")\n# Removed has_boxes filter to include ALL images\ndf[\"md5\"] = [get_md5(os.path.join(IMG_DIR, f)) for f in tqdm(df[\"filename\"], desc=\"Hashing\")]\n\ngroups = df.groupby(\"md5\")[\"filename\"].apply(list)\ndups = groups[groups.apply(len) > 1]\nprint(f\"Total: {len(df)} | Dup Groups: {len(dups)}\")\n\n# --- 2. DECIDE KEEP/DROP ---\nkeep_set, drop_list, debug_grps = set(df[\"filename\"]), [], []\n\nfor md5, fns in dups.items():\n    # Filter valid candidates (not in FORCE_DROP)\n    cands = [f for f in set(fns) if f not in FORCE_DROP]\n    if not cands: continue\n\n    # Selection Key: (1) Forced? (2) Label Count (3) Name\n    # We use min() with negative values to sort descending priority\n    winner = min(cands, key=lambda f: (\n        -int(f in FORCE_KEEP),                                      # Priority 1: Force Keep\n        -len(get_yolo(f)) if KEEP_STRATEGY==\"largest_labels\" else 0,# Priority 2: Num Boxes\n        f                                                           # Priority 3: Name (alphabetic)\n    ))\n    \n    drops = [f for f in fns if f != winner]\n    keep_set.difference_update(drops)\n    drop_list.extend(drops)\n    debug_grps.append((winner, drops))\n\nprint(f\"KEEP: {len(keep_set)} | DROP: {len(drop_list)}\")\n\n# --- 3. PLOT (OPTIONAL) ---\nif PLOT and debug_grps:\n    for i, (keep, drops) in enumerate(debug_grps[:MAX_GROUPS], 1):\n        fns = [keep] + drops\n        rows = math.ceil(len(fns)/COLS)\n        fig, axes = plt.subplots(rows, 2, figsize=(COLS*5, rows*5))\n        fig.suptitle(f\"Group #{i} | KEEP: {keep}\", y=1.01)\n        for ax, fn in zip(axes.flat, fns):\n            im = Image.open(os.path.join(IMG_DIR, fn)).convert(\"RGB\")\n            ax.imshow(im); ax.axis(\"off\"); ax.set_title(f\"{fn} [{'KEEP' if fn==keep else 'DROP'}]\")\n            for c,x,y,w,h in get_yolo(fn):\n                ax.add_patch(patches.Rectangle(((x-w/2)*im.width, (y-h/2)*im.height), w*im.width, h*im.height, fill=False, ec=\"red\", lw=2))\n        plt.tight_layout(); plt.show()\n\n# --- 4. EXPORT ---\ndf[df[\"filename\"].isin(keep_set)].reset_index(drop=True).to_csv(\"train_dedup.csv\", index=False)\npd.DataFrame(sorted(drop_list), columns=[\"dropped_filename\"]).to_csv(\"dropped_duplicates.csv\", index=False)\nprint(\"Saved: train_dedup.csv & dropped_duplicates.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:18.380291Z","iopub.execute_input":"2026-02-20T03:40:18.380661Z","iopub.status.idle":"2026-02-20T03:40:26.349868Z","shell.execute_reply.started":"2026-02-20T03:40:18.380613Z","shell.execute_reply":"2026-02-20T03:40:26.349294Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## remove overlap boxes ","metadata":{}},{"cell_type":"code","source":"import os, shutil, random, pandas as pd, matplotlib.pyplot as plt, matplotlib.patches as patches\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# --- CONFIG ---\nROOT = \"/kaggle/input/dal-shemagh-detection-challenge\"\nSRC_IMG, SRC_LBL = f\"{ROOT}/images/train\", f\"{ROOT}/labels/train\"\nDST, CSV = \"/kaggle/working/yolo_clean\", \"train_dedup.csv\"\nCLASSES, IOU_THR = [0, 1], {0: 0.1, 1: 0.1}\nPLOT_N, NAMES = 10, {0: \"Face\", 1: \"Shemagh\", 2: \"Wrong\"}\n\nfor d in [\"images/train\", \"labels/train\"]: os.makedirs(f\"{DST}/{d}\", exist_ok=True)\n\n# --- COMPACT HELPERS (Identical Logic) ---\ndef get_boxes(p):\n    if not os.path.exists(p) or os.path.getsize(p)==0: return []\n    try: return [list(map(float, l.split())) for l in open(p) if len(l.split())==5]\n    except: return []\n\ndef get_iou(a, b): # a,b = [c, x, y, w, h]\n    ax1, ay1, ax2, ay2 = a[1]-a[3]/2, a[2]-a[4]/2, a[1]+a[3]/2, a[2]+a[4]/2\n    bx1, by1, bx2, by2 = b[1]-b[3]/2, b[2]-b[4]/2, b[1]+b[3]/2, b[2]+b[4]/2\n    inter = max(0, min(ax2, bx2)-max(ax1, bx1)) * max(0, min(ay2, by2)-max(ay1, by1))\n    return inter / ((a[3]*a[4]) + (b[3]*b[4]) - inter + 1e-9)\n\ndef clean_rows(rows):\n    final = [r for r in rows if int(r[0]) not in CLASSES] # Keep ignored classes\n    for c in CLASSES:\n        # Sort by Area (w*h) Descending -> Keep Largest\n        tgts = sorted([r for r in rows if int(r[0])==c], key=lambda x: x[3]*x[4], reverse=True)\n        while tgts:\n            base = tgts.pop(0)\n            final.append(base)\n            # Remove overlapping smaller boxes\n            tgts = [t for t in tgts if get_iou(base, t) < IOU_THR[c]]\n    return final\n\n# --- MAIN PROCESS ---\nif not os.path.exists(CSV): raise FileNotFoundError(f\"Missing {CSV}\")\nfiles = sorted(pd.read_csv(CSV)[\"filename\"].unique())\nchanged = []\n\nfor fn in tqdm(files, desc=\"Cleaning\"):\n    src = f\"{SRC_IMG}/{fn}\"\n    if not os.path.exists(src): continue\n    \n    shutil.copy(src, f\"{DST}/images/train/{fn}\") # Copy image\n    \n    lbl_path = f\"{SRC_LBL}/{fn.rsplit('.',1)[0]}.txt\"\n    rows = get_boxes(lbl_path)\n    \n    if rows:\n        cleaned = clean_rows(rows)\n        with open(f\"{DST}/labels/train/{fn.rsplit('.',1)[0]}.txt\", \"w\") as f:\n            for c,x,y,w,h in cleaned: f.write(f\"{int(c)} {x:.6f} {y:.6f} {w:.6f} {h:.6f}\\n\")\n        \n        if len(cleaned) < len(rows): changed.append((fn, rows, cleaned))\n\nprint(f\"\\n✅ Done! {len(files)} processed. {len(changed)} changed.\")\nprint(f\"Output saved to: {DST}\")\n\n# --- PLOT ---\nif changed:\n    for fn, old, new in random.sample(changed, min(len(changed), PLOT_N)):\n        im = Image.open(f\"{DST}/images/train/{fn}\")\n        fig, ax = plt.subplots(1, 2, figsize=(12, 6))\n        for i, (bxs, t) in enumerate([(old, \"BEFORE\"), (new, \"AFTER\")]):\n            ax[i].imshow(im); ax[i].axis(\"off\"); ax[i].set_title(f\"{fn} | {t}: {len(bxs)}\")\n            for c,x,y,w,h in bxs:\n                x1, y1 = (x-w/2)*im.width, (y-h/2)*im.height\n                ax[i].add_patch(patches.Rectangle((x1,y1), w*im.width, h*im.height, fill=False, lw=2))\n                ax[i].text(x1, max(0, y1-5), NAMES.get(int(c),str(int(c))), fontsize=8, bbox=dict(facecolor='white', alpha=0.7))\n        plt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:26.350826Z","iopub.execute_input":"2026-02-20T03:40:26.351098Z","iopub.status.idle":"2026-02-20T03:40:34.752284Z","shell.execute_reply.started":"2026-02-20T03:40:26.351072Z","shell.execute_reply":"2026-02-20T03:40:34.751480Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## remove small boxes","metadata":{}},{"cell_type":"code","source":"import os, shutil, pandas as pd, random\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# --- CONFIG ---\nSRC_DIR = \"/kaggle/working/yolo_clean\"\nDST_DIR = \"/kaggle/working/yolo_final\"\nMIN_AREA = 0.00\nPLOT_N = 8  # Number of images to show\n\n# Setup Folders\nfor d in [\"images/train\", \"labels/train\"]: \n    os.makedirs(os.path.join(DST_DIR, d), exist_ok=True)\n\n# --- PROCESSING ---\nlabel_files = [f for f in os.listdir(f\"{SRC_DIR}/labels/train\") if f.endswith(\".txt\")]\nchanged_samples = [] # To store images for plotting\n\nprint(f\"Filtering boxes smaller than {MIN_AREA} area...\")\n\nfor lbl_fn in tqdm(label_files):\n    src_lbl = f\"{SRC_DIR}/labels/train/{lbl_fn}\"\n    dst_lbl = f\"{DST_DIR}/labels/train/{lbl_fn}\"\n    img_fn = lbl_fn.replace(\".txt\", \".jpg\") # Adjust extension if needed\n    \n    with open(src_lbl, \"r\") as f:\n        lines = f.readlines()\n    \n    kept_boxes = []\n    removed_boxes = []\n    \n    for line in lines:\n        parts = list(map(float, line.split()))\n        if len(parts) == 5:\n            area = parts[3] * parts[4] # width * height\n            if area >= MIN_AREA:\n                kept_boxes.append(parts)\n            else:\n                removed_boxes.append(parts)\n                \n    # Save cleaned labels if boxes remain\n    if kept_boxes:\n        with open(dst_lbl, \"w\") as f:\n            for b in kept_boxes:\n                f.write(f\"{int(b[0])} {b[1]:.6f} {b[2]:.6f} {b[3]:.6f} {b[4]:.6f}\\n\")\n        \n        src_img = f\"{SRC_DIR}/images/train/{img_fn}\"\n        if os.path.exists(src_img):\n            shutil.copy(src_img, f\"{DST_DIR}/images/train/{img_fn}\")\n            \n    # If we removed something, save it for the plot\n    if removed_boxes and os.path.exists(f\"{SRC_DIR}/images/train/{img_fn}\"):\n        changed_samples.append({\n            \"fn\": img_fn,\n            \"kept\": kept_boxes,\n            \"removed\": removed_boxes\n        })\n\nprint(f\"\\nDone! Found {len(changed_samples)} images with tiny boxes removed.\")\n\n# --- PLOTTING ---\nif changed_samples:\n    num_to_plot = min(len(changed_samples), PLOT_N)\n    cols = 2\n    rows = (num_to_plot + 1) // cols\n    fig, axes = plt.subplots(rows, cols, figsize=(16, rows * 6))\n    axes = axes.flatten()\n\n    for i, sample in enumerate(random.sample(changed_samples, num_to_plot)):\n        ax = axes[i]\n        im = Image.open(f\"{SRC_DIR}/images/train/{sample['fn']}\")\n        w_img, h_img = im.size\n        ax.imshow(im)\n        \n        # Plot Kept Boxes (Green)\n        for _, x, y, w, h in sample['kept']:\n            rect = patches.Rectangle(((x-w/2)*w_img, (y-h/2)*h_img), w*w_img, h*h_img, \n                                     linewidth=2, edgecolor='lime', fill=False)\n            ax.add_patch(rect)\n            \n        # Plot Removed Boxes (Red)\n        for _, x, y, w, h in sample['removed']:\n            rect = patches.Rectangle(((x-w/2)*w_img, (y-h/2)*h_img), w*w_img, h*h_img, \n                                     linewidth=2, edgecolor='red', fill=False)\n            ax.add_patch(rect)\n            ax.text((x-w/2)*w_img, (y-h/2)*h_img, f\"Tiny: {w*h:.5f}\", \n                    color='white', backgroundcolor='red', fontsize=7)\n\n        ax.set_title(f\"{sample['fn']} (Red = Removed)\")\n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:34.754292Z","iopub.execute_input":"2026-02-20T03:40:34.754732Z","iopub.status.idle":"2026-02-20T03:40:34.878865Z","shell.execute_reply.started":"2026-02-20T03:40:34.754706Z","shell.execute_reply":"2026-02-20T03:40:34.878253Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# split and chose ","metadata":{}},{"cell_type":"code","source":"import os, shutil, hashlib\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom tqdm import tqdm\n\n# -----------------------------\n# CONFIG\n# -----------------------------\nCLEAN_ROOT   = \"/kaggle/working/yolo_final\"\nORIG_ROOT    = \"/kaggle/input/dal-shemagh-detection-challenge\"\nOUT_DIR      = \"/kaggle/working/yolo_ready\"\n\nVAL_RATIO    = 0.20\nSEED         = 42\n\n# Hard requirement: EXACT #positives in VAL (if enough exist)\nMIN_VAL_POS  = 2\n\n# (Optional) only affects reporting/ordering preference, not strict\nMIN_VAL_FACE  = 0\nMIN_VAL_WRONG = 0\nMIN_VAL_CORR  = 0\n\n# Label IDs in ORIGINAL labels (before mapping)\nFACE_CLASS = 0\nSHEMAGH_CLASSES = {1, 2}\n\n# -----------------------------\n# Setup output dirs\n# -----------------------------\nfor d in [\"images/train\", \"images/val\", \"labels/train\", \"labels/val\"]:\n    os.makedirs(os.path.join(OUT_DIR, d), exist_ok=True)\n\n# -----------------------------\n# HELPERS\n# -----------------------------\ndef md5_file(path, chunk=1 << 20):\n    h = hashlib.md5()\n    with open(path, \"rb\") as f:\n        while True:\n            b = f.read(chunk)\n            if not b:\n                break\n            h.update(b)\n    return h.hexdigest()\n\ndef pick_src_img(fn):\n    p = os.path.join(CLEAN_ROOT, \"images/train\", fn)\n    if os.path.exists(p):\n        return p\n    return os.path.join(ORIG_ROOT, \"images/train\", fn)\n\ndef pick_src_lbl(stem):\n    p = os.path.join(CLEAN_ROOT, \"labels/train\", f\"{stem}.txt\")\n    if os.path.exists(p):\n        return p\n    return os.path.join(ORIG_ROOT, \"labels/train\", f\"{stem}.txt\")\n\ndef read_yolo_label(path):\n    \"\"\"Return list of (cls,x,y,w,h) floats; empty if missing/bad.\"\"\"\n    if not os.path.exists(path):\n        return []\n    rows = []\n    try:\n        with open(path, \"r\") as f:\n            for line in f:\n                parts = line.strip().split()\n                if len(parts) != 5:\n                    continue\n                rows.append(tuple(map(float, parts)))\n    except Exception:\n        return []\n    return rows\n\ndef per_image_flags(fn, right_place):\n    \"\"\"Compute flags from ORIGINAL label file + right_place.\"\"\"\n    stem = Path(fn).stem\n    rows = read_yolo_label(pick_src_lbl(stem))\n\n    n_face = sum(int(r[0]) == FACE_CLASS for r in rows)\n    n_shem = sum(int(r[0]) in SHEMAGH_CLASSES for r in rows)\n\n    has_face = int(n_face > 0)\n    has_shem = int(n_shem > 0)\n\n    # \"wrong_shemagh\" = shemagh exists but image is not right_place\n    has_wrong = int(has_shem == 1 and right_place == 0)\n    has_corr  = int(has_shem == 1 and right_place == 1)\n\n    return {\n        \"has_face\": has_face,\n        \"has_shemagh\": has_shem,\n        \"has_wrong_shemagh\": has_wrong,\n        \"has_correct_shemagh\": has_corr,\n        \"n_face\": n_face,\n        \"n_shem\": n_shem\n    }\n\n# -----------------------------\n# 1) Load meta (right_place) + build md5-dedup dataframe\n# -----------------------------\ndf_meta = pd.read_csv(f\"{ORIG_ROOT}/train_labels.csv\").drop_duplicates(\"filename\")\nrp_map = dict(zip(df_meta[\"filename\"].astype(str), df_meta[\"right_place\"].fillna(0).astype(int)))\n\nrecords = []\nseen_hashes = {}\n\nimg_dir = os.path.join(ORIG_ROOT, \"images/train\")\nall_imgs = sorted(os.listdir(img_dir))\n\nfor fn in tqdm(all_imgs, desc=\"Dedup + metadata\"):\n    src_img = pick_src_img(fn)\n    if not os.path.exists(src_img):\n        continue\n\n    h = md5_file(src_img)\n    if h in seen_hashes:\n        continue\n    seen_hashes[h] = fn\n\n    rp = int(rp_map.get(fn, 0))\n    flags = per_image_flags(fn, rp)\n\n    records.append({\n        \"filename\": fn,\n        \"src_img\": src_img,\n        \"right_place\": rp,\n        **flags\n    })\n\nall_df = pd.DataFrame(records).reset_index(drop=True)\nN = len(all_df)\nprint(\"Unique (after md5 dedup):\", N)\n\n# -----------------------------\n# 2) BETTER SPLIT\n#    - hard-cap positives in VAL to exactly MIN_VAL_POS\n#    - stratify negatives by (has_face, has_shemagh)\n# -----------------------------\nrng = np.random.default_rng(SEED)\n\nval_size = int(round(N * VAL_RATIO))\nval_size = max(val_size, MIN_VAL_POS)  # must fit positives\n\npos_df = all_df[all_df[\"right_place\"] == 1].copy()\nneg_df = all_df[all_df[\"right_place\"] == 0].copy()\n\n# A) pick EXACT positives for VAL (prefer those that actually have shemagh + face)\npos_df = pos_df.sample(frac=1, random_state=SEED).reset_index(drop=True)\npos_df = pos_df.sort_values(by=[\"has_correct_shemagh\", \"has_face\"], ascending=[False, False]).reset_index(drop=True)\n\nVAL_POS_TARGET = min(MIN_VAL_POS, len(pos_df))\nval_pos   = pos_df.iloc[:VAL_POS_TARGET]\ntrain_pos = pos_df.iloc[VAL_POS_TARGET:]\n\n# Remaining slots for VAL from negatives\nval_neg_target = val_size - len(val_pos)\nval_neg_target = min(val_neg_target, len(neg_df))\n\n# B) stratify negatives\nneg_df = neg_df.copy()\nneg_df[\"stratum\"] = (\n    neg_df[\"has_face\"].astype(int).astype(str) + \"_\" +\n    neg_df[\"has_shemagh\"].astype(int).astype(str)\n)\n\nsizes = neg_df[\"stratum\"].value_counts().to_dict()\nstrata = sorted(sizes.keys())\ncounts = np.array([sizes[s] for s in strata], dtype=float)\nproportions = counts / counts.sum() if counts.sum() > 0 else np.zeros_like(counts)\n\nraw = proportions * val_neg_target\nalloc = np.floor(raw).astype(int)\nrema = raw - alloc\n\n# Fix rounding: sum(alloc) must equal val_neg_target\ndiff = int(val_neg_target - alloc.sum())\nif diff > 0:\n    for i in np.argsort(-rema)[:diff]:\n        alloc[i] += 1\nelif diff < 0:\n    order = np.argsort(rema)\n    j = 0\n    while diff < 0 and j < len(order):\n        i = order[j]\n        if alloc[i] > 0:\n            alloc[i] -= 1\n            diff += 1\n        j += 1\n\nval_neg_parts, train_neg_parts = [], []\nfor s, k in zip(strata, alloc):\n    sub = neg_df[neg_df[\"stratum\"] == s].sample(frac=1, random_state=SEED)\n    k = int(min(k, len(sub)))\n    val_neg_parts.append(sub.iloc[:k])\n    train_neg_parts.append(sub.iloc[k:])\n\nval_neg   = pd.concat(val_neg_parts, axis=0) if len(val_neg_parts) else neg_df.iloc[0:0]\ntrain_neg = pd.concat(train_neg_parts, axis=0) if len(train_neg_parts) else neg_df\n\n# C) combine\ntrain_final = pd.concat([train_pos, train_neg], axis=0).sample(frac=1, random_state=SEED).reset_index(drop=True)\nval_final   = pd.concat([val_pos,   val_neg],   axis=0).sample(frac=1, random_state=SEED).reset_index(drop=True)\n\n# D) report\ndef ratio(df, col):\n    return float(df[col].mean()) if len(df) else 0.0\n\nprint(\"\\nSplit summary:\")\nprint(\"TOTAL:\", len(all_df),\n      \"| pos:\", int(all_df[\"right_place\"].sum()),\n      \"| face:\", int(all_df[\"has_face\"].sum()),\n      \"| shem:\", int(all_df[\"has_shemagh\"].sum()),\n      \"| wrong_shem:\", int(all_df[\"has_wrong_shemagh\"].sum()))\nprint(\"TRAIN:\", len(train_final),\n      \"| pos:\", int(train_final[\"right_place\"].sum()),\n      \"| face:\", int(train_final[\"has_face\"].sum()),\n      \"| shem:\", int(train_final[\"has_shemagh\"].sum()),\n      \"| wrong_shem:\", int(train_final[\"has_wrong_shemagh\"].sum()))\nprint(\"VAL  :\", len(val_final),\n      \"| pos:\", int(val_final[\"right_place\"].sum()),\n      \"| face:\", int(val_final[\"has_face\"].sum()),\n      \"| shem:\", int(val_final[\"has_shemagh\"].sum()),\n      \"| wrong_shem:\", int(val_final[\"has_wrong_shemagh\"].sum()))\n\nprint(\"\\nVAL ratios vs TOTAL:\")\nprint(\" right_place   total=\", ratio(all_df, \"right_place\"),   \"| val=\", ratio(val_final, \"right_place\"))\nprint(\" has_face      total=\", ratio(all_df, \"has_face\"),      \"| val=\", ratio(val_final, \"has_face\"))\nprint(\" has_shemagh   total=\", ratio(all_df, \"has_shemagh\"),   \"| val=\", ratio(val_final, \"has_shemagh\"))\nprint(\" wrong_shemagh total=\", ratio(all_df, \"has_wrong_shemagh\"), \"| val=\", ratio(val_final, \"has_wrong_shemagh\"))\n\n# -----------------------------\n# 3) Copy files + map classes\n# -----------------------------\ndef finalize_split(df, split):\n    for _, row in tqdm(df.iterrows(), total=len(df), desc=f\"Writing {split}\"):\n        fn, stem = row[\"filename\"], Path(row[\"filename\"]).stem\n\n        # copy image\n        shutil.copy2(row[\"src_img\"], f\"{OUT_DIR}/images/{split}/{fn}\")\n\n        # pick label\n        src_lbl = pick_src_lbl(stem)\n        dst_lbl = f\"{OUT_DIR}/labels/{split}/{stem}.txt\"\n\n        if os.path.exists(src_lbl):\n            with open(src_lbl, \"r\") as f_in, open(dst_lbl, \"w\") as f_out:\n                for line in f_in:\n                    parts = line.split()\n                    if not parts:\n                        continue\n                    c = parts[0]\n\n                    # Map:\n                    #   face stays 0\n                    #   shemagh -> 1 if right_place==1 else 2\n                    new_c = 0 if c == str(FACE_CLASS) else (1 if int(row[\"right_place\"]) == 1 else 2)\n\n                    f_out.write(f\"{new_c} {' '.join(parts[1:])}\\n\")\n        else:\n            open(dst_lbl, \"w\").close()\n\nfinalize_split(train_final, \"train\")\nfinalize_split(val_final, \"val\")\n\n# -----------------------------\n# 4) Save split CSVs\n# -----------------------------\ntrain_final.to_csv(os.path.join(OUT_DIR, \"split_train.csv\"), index=False)\nval_final.to_csv(os.path.join(OUT_DIR, \"split_val.csv\"), index=False)\n\nprint(\"\\n✅ Split Done!\")\nprint(f\"VAL positives (right_place==1): {int(val_final['right_place'].sum())} (target={MIN_VAL_POS})\")\nprint(f\"VAL has_face: {int(val_final['has_face'].sum())}\")\nprint(f\"VAL has_shemagh: {int(val_final['has_shemagh'].sum())}\")\nprint(f\"VAL wrong_shemagh: {int(val_final['has_wrong_shemagh'].sum())}\")\nprint(\"Saved splits to:\", OUT_DIR)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:34.879789Z","iopub.execute_input":"2026-02-20T03:40:34.880074Z","iopub.status.idle":"2026-02-20T03:40:36.005255Z","shell.execute_reply.started":"2026-02-20T03:40:34.880050Z","shell.execute_reply":"2026-02-20T03:40:36.004536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Define the absolute path for the YAML\nyaml_path = os.path.join(OUT_DIR, \"data.yaml\")\n\n# Construct the YAML content\n# Using absolute paths is usually safer for YOLO training in Kaggle/Colab\nyaml_content = f\"\"\"\npath: {os.path.abspath(OUT_DIR)}\ntrain: images/train\nval: images/val\n\nnc: 3\nnames: ['Face', 'Shemagh_Correct', 'Shemagh_Wrong']\n\"\"\"\n\nwith open(yaml_path, \"w\") as f:\n    f.write(yaml_content.strip())\n\nprint(f\"✅ data.yaml created at: {yaml_path}\")\nprint(\"--- YAML Content ---\")\nprint(yaml_content)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:36.006265Z","iopub.execute_input":"2026-02-20T03:40:36.006492Z","iopub.status.idle":"2026-02-20T03:40:36.011468Z","shell.execute_reply.started":"2026-02-20T03:40:36.006469Z","shell.execute_reply":"2026-02-20T03:40:36.010948Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# simple offline","metadata":{}},{"cell_type":"code","source":"import os, cv2, random, hashlib, warnings\nimport numpy as np\nimport albumentations as A\nfrom tqdm import tqdm\nfrom pathlib import Path\n\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"albumentations\")\n\n# ---- CONFIG (Matches your previous cells) ----\nWORK_DIR = \"/kaggle/working/yolo_ready\"\nTRAIN_IMG_DIR = os.path.join(WORK_DIR, \"images/train\")\nTRAIN_LBL_DIR = os.path.join(WORK_DIR, \"labels/train\")\nVAL_IMG_DIR   = os.path.join(WORK_DIR, \"images/val\")\nVAL_LBL_DIR   = os.path.join(WORK_DIR, \"labels/val\")\n\n# Augmentation Knobs\nTRAIN_AUG_POS_N = 8\nTRAIN_AUG_NEG_N = 0\nVAL_AUG_POS_N   = 9\nVAL_AUG_NEG_N   = 1\n\nTR_POS_SUFFIX, TR_NEG_SUFFIX = \"trpaug\", \"trnaug\"\nVL_POS_SUFFIX, VL_NEG_SUFFIX = \"vlpaug\", \"vlnaug\"\n\ntrain_count_before = sum(1 for f in os.listdir(TRAIN_IMG_DIR) if f.lower().endswith((\".jpg\",\".jpeg\",\".png\")))\nval_count_before   = sum(1 for f in os.listdir(VAL_IMG_DIR)   if f.lower().endswith((\".jpg\",\".jpeg\",\".png\")))\n\nEPS = 1e-6\nREPRO_SEED = 42\n\n# ---- FACE BLUR KNOBS ----\nFACE_BLUR_P = 0.0\nFACE_BLUR_MAX_K = 7\nFACE_CLASS_ID = 0  # 0 = Face\n\n# ---- POS->NEG paste knobs (Val ONLY) ----\nPOS_TO_NEG_P = 0.0             # 30% chance per POS augmented sample\nSHEMAGH_CLASS_IDS = {1, 2}       # adjust if needed (e.g. {1} only)\nCP_PAD_FRAC_RANGE = (0.10, 0.25) # padding after union crop\nCP_FEATHER_K = 11\nCP_SCALE_RANGE = (0.90, 1.10)\nCP_MAX_PLACE_TRIES = 120\nCP_OVERLAP_EPS_PX2 = 1.0         # strict: any overlap area > eps means reject placement\nDONOR_INCLUDE_MIN_IOA = 0.90     # include donor boxes fully inside crop (IOA >= 0.90)\n\n# ============================================================\n# HELPERS\n# ============================================================\n\ndef _odd(x):\n    x = int(x)\n    return x if x % 2 == 1 else x + 1\n\ndef blur_faces_in_image(img_rgb, bboxes_yolo, labels, p=0.35, max_k=31, face_cls=0):\n    if random.random() >= p:\n        return img_rgb\n    H, W = img_rgb.shape[:2]\n    out = img_rgb.copy()\n    for (x, y, w, h), lab in zip(bboxes_yolo, labels):\n        if int(lab) != int(face_cls):\n            continue\n        x1 = int((x - w/2) * W); y1 = int((y - h/2) * H)\n        x2 = int((x + w/2) * W); y2 = int((y + h/2) * H)\n        x1 = max(0, min(W-1, x1)); x2 = max(0, min(W, x2))\n        y1 = max(0, min(H-1, y1)); y2 = max(0, min(H, y2))\n        if x2 - x1 < 3 or y2 - y1 < 3:\n            continue\n        crop = out[y1:y2, x1:x2]\n        k = int(0.25 * min(crop.shape[0], crop.shape[1]))\n        k = max(3, _odd(k))\n        k = min(k, max_k)\n        k = min(k, _odd(min(crop.shape[0]-1, crop.shape[1]-1)))\n        if k < 3:\n            continue\n        out[y1:y2, x1:x2] = cv2.GaussianBlur(crop, (k, k), 0)\n    return out\n\ndef read_yolo_label(lbl_path):\n    bboxes, labels = [], []\n    if not os.path.exists(lbl_path) or os.path.getsize(lbl_path) == 0:\n        return bboxes, labels\n    for line in open(lbl_path, \"r\"):\n        p = line.strip().split()\n        if len(p) != 5:\n            continue\n        cls = int(float(p[0]))\n        x, y, w, h = map(float, p[1:])\n        bboxes.append([x, y, w, h])\n        labels.append(cls)\n    return bboxes, labels\n\ndef sanitize_yolo_bboxes(bboxes, labels, eps=1e-6):\n    out_b, out_l = [], []\n    for (x, y, w, h), lab in zip(bboxes, labels):\n        x = min(max(x, eps), 1 - eps)\n        y = min(max(y, eps), 1 - eps)\n        max_w, max_h = 2 * min(x, 1 - x) - eps, 2 * min(y, 1 - y) - eps\n        w, h = min(w, max_w), min(h, max_h)\n        if w > eps and h > eps:\n            out_b.append([x, y, w, h]); out_l.append(lab)\n    return out_b, out_l\n\ndef write_yolo_label(lbl_path, bboxes, labels):\n    os.makedirs(os.path.dirname(lbl_path), exist_ok=True)\n    with open(lbl_path, \"w\") as f:\n        for lab, (x, y, w, h) in zip(labels, bboxes):\n            f.write(f\"{int(lab)} {x:.6f} {y:.6f} {w:.6f} {h:.6f}\\n\")\n\ndef stable_seed(*parts, base=42):\n    s = \"|\".join(map(str, parts))\n    h = hashlib.md5(s.encode(\"utf-8\")).hexdigest()\n    return (int(h[:8], 16) + int(base)) % (2**32 - 1)\n\n# ---- bbox math (pixel xyxy) ----\ndef yolo_to_xyxy_px(bb, W, H):\n    x, y, w, h = bb\n    x1 = (x - w/2) * W\n    y1 = (y - h/2) * H\n    x2 = (x + w/2) * W\n    y2 = (y + h/2) * H\n    return np.array([x1, y1, x2, y2], dtype=np.float32)\n\ndef xyxy_px_to_yolo(bb, W, H, eps=1e-6):\n    x1, y1, x2, y2 = map(float, bb)\n    x1 = max(0.0, min(x1, W-1.0)); y1 = max(0.0, min(y1, H-1.0))\n    x2 = max(0.0, min(x2, W-1.0)); y2 = max(0.0, min(y2, H-1.0))\n    if x2 <= x1 or y2 <= y1:\n        return None\n    w = x2 - x1\n    h = y2 - y1\n    x = x1 + w/2\n    y = y1 + h/2\n    xn, yn, wn, hn = x/W, y/H, w/W, h/H\n    xn = min(max(xn, eps), 1-eps); yn = min(max(yn, eps), 1-eps)\n    wn = min(max(wn, eps), 1-eps); hn = min(max(hn, eps), 1-eps)\n    return [xn, yn, wn, hn]\n\ndef box_area_xyxy(b):\n    x1,y1,x2,y2 = b\n    return max(0.0, x2-x1) * max(0.0, y2-y1)\n\ndef intersect_area_xyxy(a, b):\n    ax1,ay1,ax2,ay2 = a\n    bx1,by1,bx2,by2 = b\n    x1 = max(ax1, bx1); y1 = max(ay1, by1)\n    x2 = min(ax2, bx2); y2 = min(ay2, by2)\n    return max(0.0, x2-x1) * max(0.0, y2-y1)\n\ndef iou_xyxy(a, b):\n    inter = intersect_area_xyxy(a, b)\n    if inter <= 0:\n        return 0.0\n    ua = box_area_xyxy(a) + box_area_xyxy(b) - inter\n    return float(inter / (ua + 1e-9))\n\ndef feather_mask(mask01, k=11):\n    k = int(k)\n    if k <= 1:\n        return mask01\n    if k % 2 == 0:\n        k += 1\n    blur = cv2.GaussianBlur(mask01, (k,k), 0)\n    mn, mx = float(blur.min()), float(blur.max())\n    if mx > mn:\n        blur = (blur - mn) / (mx - mn)\n    return blur\n\ndef has_any_class(lbl_path, class_ids):\n    _, labs = read_yolo_label(lbl_path)\n    return any(int(c) in class_ids for c in labs)\n\n# ============================================================\n# POS -> NEG background paste (FIXED: includes face label)\n# - pick one shemagh box (donor POS)\n# - find the \"best\" face box in donor (max IoU with shemagh OR nearest)\n# - crop = UNION(shemagh, face) then pad\n# - paste patch to random NEG bg with strict non-overlap vs bg GT\n# - add labels for ALL donor boxes inside crop (IOA>=0.90), including face+shemagh\n# ============================================================\ndef paste_shemagh_with_face_into_neg_bg(\n    neg_img_rgb, neg_bbs, neg_labs,\n    pos_img_rgb, pos_bbs, pos_labs,\n):\n    H, W   = neg_img_rgb.shape[:2]\n    Hd, Wd = pos_img_rgb.shape[:2]\n\n    # choose one shemagh bbox index from POS\n    shem_idxs = [i for i,c in enumerate(pos_labs) if int(c) in SHEMAGH_CLASS_IDS]\n    if len(shem_idxs) == 0:\n        return None\n    j = random.choice(shem_idxs)\n\n    # donor boxes xyxy\n    d_xyxy = [yolo_to_xyxy_px(bb, Wd, Hd) for bb in pos_bbs]\n    shem_box = d_xyxy[j]\n    sx1,sy1,sx2,sy2 = shem_box\n    if (sx2 - sx1) < 2 or (sy2 - sy1) < 2:\n        return None\n\n    # find best face in donor\n    face_idxs = [i for i,c in enumerate(pos_labs) if int(c) == int(FACE_CLASS_ID)]\n    face_box = None\n    if len(face_idxs) > 0:\n        # pick face with max IoU to shemagh, fallback to nearest center\n        best_iou = -1.0\n        best_i = None\n        scx, scy = (sx1+sx2)/2, (sy1+sy2)/2\n        best_dist = 1e18\n        for fi in face_idxs:\n            fb = d_xyxy[fi]\n            v = iou_xyxy(fb, shem_box)\n            if v > best_iou:\n                best_iou = v\n                best_i = fi\n        if best_iou <= 0.0:\n            for fi in face_idxs:\n                fb = d_xyxy[fi]\n                fcx, fcy = (fb[0]+fb[2])/2, (fb[1]+fb[3])/2\n                d = (fcx-scx)**2 + (fcy-scy)**2\n                if d < best_dist:\n                    best_dist = d\n                    best_i = fi\n        face_box = d_xyxy[best_i] if best_i is not None else None\n\n    # crop base = union(shemagh, face if exists)\n    if face_box is not None:\n        x1 = min(shem_box[0], face_box[0])\n        y1 = min(shem_box[1], face_box[1])\n        x2 = max(shem_box[2], face_box[2])\n        y2 = max(shem_box[3], face_box[3])\n        base_crop = np.array([x1,y1,x2,y2], dtype=np.float32)\n    else:\n        base_crop = shem_box.copy()\n\n    # pad crop\n    cw = base_crop[2] - base_crop[0]\n    ch = base_crop[3] - base_crop[1]\n    pad_frac = random.uniform(*CP_PAD_FRAC_RANGE)\n    pad = pad_frac * max(cw, ch)\n\n    crop = np.array([base_crop[0]-pad, base_crop[1]-pad, base_crop[2]+pad, base_crop[3]+pad], dtype=np.float32)\n    crop[0] = max(0.0, min(crop[0], Wd-1.0))\n    crop[1] = max(0.0, min(crop[1], Hd-1.0))\n    crop[2] = max(0.0, min(crop[2], Wd-1.0))\n    crop[3] = max(0.0, min(crop[3], Hd-1.0))\n    if crop[2] <= crop[0] or crop[3] <= crop[1]:\n        return None\n    cx1,cy1,cx2,cy2 = crop\n\n    patch = pos_img_rgb[int(cy1):int(cy2), int(cx1):int(cx2)].copy()\n    if patch.size == 0:\n        return None\n\n    # target boxes in xyxy for strict overlap (bg GT)\n    tgt_xyxy = [yolo_to_xyxy_px(bb, W, H) for bb in neg_bbs]\n\n    # include ALL donor boxes that are (mostly) inside crop (IOA>=0.90)\n    inside = []\n    for c, b in zip(pos_labs, d_xyxy):\n        a = box_area_xyxy(b)\n        if a <= 1:\n            continue\n        ioa = intersect_area_xyxy(b, crop) / (a + 1e-9)\n        if ioa >= DONOR_INCLUDE_MIN_IOA:\n            inside.append((int(c), b.copy()))\n    # safety: force include shemagh and face if found (even if IOA misses due to float)\n    inside.append((int(pos_labs[j]), shem_box.copy()))\n    if face_box is not None:\n        inside.append((int(FACE_CLASS_ID), face_box.copy()))\n\n    # de-dup inside by (cls, rounded coords)\n    uniq = {}\n    for c, b in inside:\n        key = (c, int(b[0]), int(b[1]), int(b[2]), int(b[3]))\n        uniq[key] = (c, b)\n    inside = list(uniq.values())\n\n    # scale patch\n    s = random.uniform(*CP_SCALE_RANGE)\n    ph, pw = patch.shape[:2]\n    pw2 = max(2, int(pw*s)); ph2 = max(2, int(ph*s))\n    patch = cv2.resize(patch, (pw2, ph2), interpolation=cv2.INTER_LINEAR)\n    ph, pw = patch.shape[:2]\n\n    # fit patch in target image\n    if pw >= W or ph >= H:\n        s2 = min((W-2)/max(1,pw), (H-2)/max(1,ph))\n        pw2 = max(2, int(pw*s2)); ph2 = max(2, int(ph*s2))\n        patch = cv2.resize(patch, (pw2, ph2), interpolation=cv2.INTER_LINEAR)\n        ph, pw = patch.shape[:2]\n        s *= s2\n\n    # strict placement: patch region doesn't overlap ANY target GT box\n    chosen = None\n    for _ in range(CP_MAX_PLACE_TRIES):\n        px1 = random.randint(0, W - pw)\n        py1 = random.randint(0, H - ph)\n        px2 = px1 + pw\n        py2 = py1 + ph\n        patch_region = np.array([px1,py1,px2,py2], dtype=np.float32)\n\n        total_ov = 0.0\n        for tb in tgt_xyxy:\n            total_ov += intersect_area_xyxy(patch_region, tb)\n\n        if total_ov <= CP_OVERLAP_EPS_PX2:\n            chosen = (px1, py1, patch_region)\n            break\n    if chosen is None:\n        return None\n    px1, py1, patch_region = chosen\n\n    # blend patch onto background\n    out = neg_img_rgb.astype(np.float32).copy()\n    patch_f = patch.astype(np.float32)\n\n    mask = np.zeros((H, W), dtype=np.float32)\n    x1r,y1r,x2r,y2r = map(int, patch_region)\n    mask[y1r:y2r, x1r:x2r] = 1.0\n    mask = feather_mask(mask, k=CP_FEATHER_K)\n    alpha = mask[..., None]\n\n    patch_canvas = np.zeros_like(out, dtype=np.float32)\n    patch_canvas[y1r:y2r, x1r:x2r] = patch_f\n    out = (out * (1 - alpha) + patch_canvas * alpha).astype(np.uint8)\n\n    # map all inside boxes to target coords\n    new_bbs = list(neg_bbs)\n    new_labs = list(neg_labs)\n\n    for c, b in inside:\n        bx1,by1,bx2,by2 = b\n        # relative to crop -> scaled -> translated\n        rx1 = (bx1 - cx1) * s\n        ry1 = (by1 - cy1) * s\n        rx2 = (bx2 - cx1) * s\n        ry2 = (by2 - cy1) * s\n        tb = np.array([px1+rx1, py1+ry1, px1+rx2, py1+ry2], dtype=np.float32)\n        ybb = xyxy_px_to_yolo(tb, W, H, eps=EPS)\n        if ybb is not None:\n            new_bbs.append(ybb)\n            new_labs.append(int(c))\n\n    new_bbs, new_labs = sanitize_yolo_bboxes(new_bbs, new_labs, eps=EPS)\n    if len(new_bbs) == 0:\n        return None\n    return out, new_bbs, new_labs\n\n# ============================================================\n# AUG PIPELINES\n# ============================================================\ntrain_aug_pos = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightnessContrast(p=0.4),\n    A.HueSaturationValue(p=0.3),\n], bbox_params=A.BboxParams(format=\"yolo\", label_fields=[\"labels\"], min_visibility=0.15))\n\ntrain_aug_neg = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightnessContrast(p=0.3),\n], bbox_params=A.BboxParams(format=\"yolo\", label_fields=[\"labels\"], min_visibility=0.15))\n\n# ---- VARIABLE NAME FIX ----\ntrain_pos_set = set(train_final[train_final[\"right_place\"] == 1][\"filename\"].tolist())\ntrain_neg_set = set(train_final[train_final[\"right_place\"] == 0][\"filename\"].tolist())\nval_pos_set   = set(val_final[val_final[\"right_place\"] == 1][\"filename\"].tolist())\nval_neg_set   = set(val_final[val_final[\"right_place\"] == 0][\"filename\"].tolist())\n\ndef augment_posneg_folder(img_dir, lbl_dir, pos_set, neg_set, n_pos, n_neg,\n                          suffix_pos, suffix_neg, aug_pos, aug_neg, split_tag):\n\n    imgs = sorted([f for f in os.listdir(img_dir)\n                   if f.lower().endswith((\".jpg\",\".jpeg\",\".png\"))\n                   and suffix_pos not in f and suffix_neg not in f])\n\n    # TRAIN only: NEG BACKGROUND pool = right_place==0 and contains NO shemagh boxes\n    neg_bg_pool = []\n    if split_tag == \"train\":\n        for fn_bg in imgs:\n            if fn_bg not in neg_set:\n                continue\n            stem_bg = Path(fn_bg).stem\n            lbl_bg = os.path.join(lbl_dir, stem_bg + \".txt\")\n            if not has_any_class(lbl_bg, SHEMAGH_CLASS_IDS):\n                neg_bg_pool.append(fn_bg)\n\n    made = bad = 0\n    for fn in tqdm(imgs, desc=f\"Augmenting {split_tag}\"):\n        stem = Path(fn).stem\n        img_path = os.path.join(img_dir, fn)\n        lbl_path = os.path.join(lbl_dir, stem + \".txt\")\n\n        is_pos = fn in pos_set\n        n_per, aug_comp, suffix = (n_pos, aug_pos, suffix_pos) if is_pos else (n_neg, aug_neg, suffix_neg)\n        if n_per == 0:\n            continue\n\n        img = cv2.imread(img_path)\n        if img is None:\n            bad += 1\n            continue\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        bboxes, labels = read_yolo_label(lbl_path)\n        bboxes, labels = sanitize_yolo_bboxes(bboxes, labels, eps=EPS)\n        if not bboxes:\n            continue\n\n        for i in range(n_per):\n            seed = stable_seed(split_tag, fn, suffix, i, base=REPRO_SEED)\n            random.seed(seed); np.random.seed(seed)\n\n            base_img = img\n            base_bbs = bboxes\n            base_labs = labels\n\n            # ---------------------------------------------------------\n            # TRAIN only: 30% of POS samples use NEG background with pasted shemagh+face\n            # ---------------------------------------------------------\n            if split_tag == \"train\" and is_pos and len(neg_bg_pool) > 0:\n                if random.random() < POS_TO_NEG_P:\n                    bg_fn = random.choice(neg_bg_pool)\n                    bg_stem = Path(bg_fn).stem\n                    bg_img_path = os.path.join(img_dir, bg_fn)\n                    bg_lbl_path = os.path.join(lbl_dir, bg_stem + \".txt\")\n\n                    bg_img = cv2.imread(bg_img_path)\n                    if bg_img is not None:\n                        bg_img = cv2.cvtColor(bg_img, cv2.COLOR_BGR2RGB)\n                        bg_bbs, bg_labs = read_yolo_label(bg_lbl_path)\n                        bg_bbs, bg_labs = sanitize_yolo_bboxes(bg_bbs, bg_labs, eps=EPS)\n\n                        res = paste_shemagh_with_face_into_neg_bg(\n                            neg_img_rgb=bg_img, neg_bbs=bg_bbs, neg_labs=bg_labs,\n                            pos_img_rgb=img,    pos_bbs=bboxes, pos_labs=labels,\n                        )\n                        if res is not None:\n                            base_img, base_bbs, base_labs = res\n\n            if not base_bbs:\n                continue\n\n            try:\n                t = aug_comp(image=base_img, bboxes=base_bbs, labels=base_labs)\n                aug_img, aug_bbs, aug_labs = t[\"image\"], list(t[\"bboxes\"]), list(t[\"labels\"])\n                if not aug_bbs:\n                    continue\n\n                # face blur AFTER aug (now pasted faces are labeled too)\n                aug_img = blur_faces_in_image(\n                    aug_img, aug_bbs, aug_labs,\n                    p=FACE_BLUR_P,\n                    max_k=FACE_BLUR_MAX_K,\n                    face_cls=FACE_CLASS_ID\n                )\n\n                new_stem = f\"{stem}_{suffix}_{i:04d}\"\n                new_img_path = os.path.join(img_dir, new_stem + \".jpg\")\n                new_lbl_path = os.path.join(lbl_dir, new_stem + \".txt\")\n\n                cv2.imwrite(new_img_path, cv2.cvtColor(aug_img, cv2.COLOR_RGB2BGR))\n                write_yolo_label(new_lbl_path, aug_bbs, aug_labs)\n                made += 1\n            except:\n                continue\n\n    return made, bad\nval_aug_pos = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.ChannelShuffle(p=0.15),\n    A.RandomBrightnessContrast(p=0.4),\n    A.ToGray(p=0.1),\n    A.HueSaturationValue(p=0.3),\n], bbox_params=A.BboxParams(format=\"yolo\", label_fields=[\"labels\"], min_visibility=0.15))\n\n\n\n# ---- RUN ----\ntr_made, tr_bad = augment_posneg_folder(\n    TRAIN_IMG_DIR, TRAIN_LBL_DIR,\n    train_pos_set, train_neg_set,\n    TRAIN_AUG_POS_N, TRAIN_AUG_NEG_N,\n    TR_POS_SUFFIX, TR_NEG_SUFFIX,\n    train_aug_pos, train_aug_pos,\n    \"train\"\n)\n\n# COPY-PASTE OFF for val (split_tag != \"train\")\nvl_made, vl_bad = augment_posneg_folder(\n    VAL_IMG_DIR, VAL_LBL_DIR,\n    val_pos_set, val_neg_set,\n    VAL_AUG_POS_N, VAL_AUG_NEG_N,\n    VL_POS_SUFFIX, VL_NEG_SUFFIX,\n    val_aug_pos, val_aug_pos,\n    \"val\"\n)\n\ntrain_count_after = sum(1 for f in os.listdir(TRAIN_IMG_DIR) if f.lower().endswith((\".jpg\",\".jpeg\",\".png\")))\nval_count_after   = sum(1 for f in os.listdir(VAL_IMG_DIR)   if f.lower().endswith((\".jpg\",\".jpeg\",\".png\")))\n\nprint(\"\\n\" + \"=\"*40)\nprint(f\"{'Split':<10} | {'Before':<10} | {'After':<10} | {'Added':<10}\")\nprint(\"-\" * 40)\nprint(f\"{'TRAIN':<10} | {train_count_before:<10} | {train_count_after:<10} | +{tr_made}\")\nprint(f\"{'VAL':<10} | {val_count_before:<10} | {val_count_after:<10} | +{vl_made}\")\nprint(\"=\"*40)\n\nif tr_bad > 0 or vl_bad > 0:\n    print(f\"⚠️ Warning: Skipped {tr_bad + vl_bad} unreadable images.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T03:40:57.883077Z","iopub.execute_input":"2026-02-20T03:40:57.883382Z","iopub.status.idle":"2026-02-20T03:41:00.364642Z","shell.execute_reply.started":"2026-02-20T03:40:57.883353Z","shell.execute_reply":"2026-02-20T03:41:00.363999Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\nimport albumentations as A\nmodel = YOLO(\"yolo26x.pt\")\n\n\nprint(\"Starting training...\")\n\nresults = model.train(\n    data=\"/kaggle/working/yolo_ready/data.yaml\",\n    epochs=35,                \n    batch=32,                   \n    imgsz=640,\n    patience=500,\n    \n    optimizer='SGD',         \n    lr0=0.001,\n    lrf = 0.001,\n    device = [0,1],\n    weight_decay=0.0002,       \n    cos_lr=True,               \n    save_period=10,             \n    workers=8,\n    \n    close_mosaic=20,\n    hsv_h=0.02,\n    hsv_s=0.7,\n    hsv_v=0.3,\n    bgr=0.1,\n\n    \n    fliplr=0.5,\n    translate=0.1,\n    scale=0.5,\n    mixup = 0.1,\n    cutmix = 0.1,\n    \n    warmup_epochs= 3,\n    warmup_momentum= 1,\n    \n    exist_ok = True,\n    project = 'ggg',\n\n\n    freeze = 6,\n    \n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T04:56:35.355150Z","iopub.execute_input":"2026-02-20T04:56:35.355493Z","iopub.status.idle":"2026-02-20T05:11:32.075303Z","shell.execute_reply.started":"2026-02-20T04:56:35.355459Z","shell.execute_reply":"2026-02-20T05:11:32.074501Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# submisson ","metadata":{}},{"cell_type":"code","source":"import os, glob, io, contextlib, json, tempfile, itertools, time\nimport numpy as np\nimport cv2\nfrom tqdm import tqdm\n\n# ----------------------------\n# Auto-install deps (quiet)\n# ----------------------------\ntry:\n    from pycocotools.coco import COCO\n    from pycocotools.cocoeval import COCOeval\n    from ensemble_boxes import weighted_boxes_fusion\nexcept Exception:\n    os.system(\"pip -q install pycocotools ensemble-boxes\")\n    from pycocotools.coco import COCO\n    from pycocotools.cocoeval import COCOeval\n    from ensemble_boxes import weighted_boxes_fusion\n\n# ----------------------------\n# Must have a loaded YOLO model\n# ----------------------------\nassert \"model\" in globals(), \"❌ 'model' is not defined. Load Ultralytics YOLO first: model = YOLO('path.pt')\"\n\n# ==============================================================================\n# 0) SPEED BUDGET (EDIT THIS)\n# ==============================================================================\nMAX_COCO_EVAL_CALLS = 400   # <= MAIN SPEED KNOB (e.g., 200, 400, 800)\nSTOP_IF_UB_BELOW_BEST = True  # if True: stop early when remaining can't beat best\nPRINT_EVERY = 50\n\n# ==============================================================================\n# 1) CONFIG (EDIT THESE)\n# ==============================================================================\nROOT_DIR = \"/kaggle/working/yolo_ready\"\nIMG_DIR  = os.path.join(ROOT_DIR, \"images/val\")\nLBL_DIR  = os.path.join(ROOT_DIR, \"labels/val\")\n\n# Inference knobs\nPRED_CONF = 0.01\nIOU_NMS   = 0.70\nIMGSZ     = 640\nTTA_HFLIP = True\nTTA_GRAY  = True\n\n# Group-NMS knobs\nIOU_FACE       = 0.50\nIOU_GROUP_SHEM = 0.45\nCONTAIN_FACE   = 0.95\nCONTAIN_SHEM   = 0.85\nXYWH_IS_CENTER = True\n\n# Right_place + geometry override\nGEOM_OVERRIDE_ENABLED = True\nFACE_MIN_FOR_OVERRIDE = 0.10\nSHEM_MIN_FOR_OVERRIDE = 0.10\n\n# Export caps\nMAX_FACE_EXPORT = 7\nMAX_SHEM_EXPORT = 7\n\n# Require both if right_place==1\nREQUIRE_BOTH_IF_RIGHT_PLACE = True\n\n# ==============================================================================\n# 2) GRID (EDIT THESE LISTS)\n# ==============================================================================\nGRID = {\n    \"THR_RIGHT_PLACE\":        [0.005, 0.01, 0.03, 0.05],\n    \"FACE_OVERLAP_FRAC_THR\":  [0.65, 0.70, 0.75, 0.85],\n    \"FACE_HIGH_THR\":          [0.08, 0.10, 0.15, 0.20],\n    \"FACE_MIN_EXPORT\":        [0.03, 0.05, 0.10],\n    \"SHEM_HIGH_THR\":          [0.08, 0.10, 0.15, 0.20],\n    \"SHEM_MIN_EXPORT\":        [0.005, 0.01, 0.03, 0.05, 0.10, 0.15],\n    \"WBF_FACE_IOU\":           [0.45, 0.50, 0.55, 0.60],\n    \"WBF_SHEM_IOU\":           [0.45, 0.50, 0.55],\n}\n\n# ==============================================================================\n# 3) HELPERS\n# ==============================================================================\ndef stem(fn):\n    return os.path.splitext(os.path.basename(fn))[0]\n\ndef load_gt_labels(img_fn):\n    p = os.path.join(LBL_DIR, stem(img_fn) + \".txt\")\n    if not os.path.exists(p):\n        return []\n    out = []\n    with open(p, \"r\") as f:\n        for line in f:\n            t = line.strip().split()\n            if len(t) != 5:\n                continue\n            c = int(float(t[0]))\n            x, y, w, h = map(float, t[1:])\n            out.append((c, (x, y, w, h)))\n    return out\n\ndef xywh_to_xyxy(bb, is_center=True):\n    x, y, w, h = bb\n    if is_center:\n        return (x - w/2, y - h/2, x + w/2, y + h/2)\n    else:\n        return (x, y, x + w, y + h)\n\ndef iou_xywh(bb1, bb2, is_center=True):\n    ax1, ay1, ax2, ay2 = xywh_to_xyxy(bb1, is_center=is_center)\n    bx1, by1, bx2, by2 = xywh_to_xyxy(bb2, is_center=is_center)\n    ix1, iy1 = max(ax1, bx1), max(ay1, by1)\n    ix2, iy2 = min(ax2, bx2), min(ay2, by2)\n    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)\n    inter = iw * ih\n    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)\n    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)\n    union = area_a + area_b - inter\n    return 0.0 if union <= 0 else inter / union\n\ndef inter_over_min_area_xywh(bb1, bb2, is_center=True):\n    ax1, ay1, ax2, ay2 = xywh_to_xyxy(bb1, is_center=is_center)\n    bx1, by1, bx2, by2 = xywh_to_xyxy(bb2, is_center=is_center)\n    ix1, iy1 = max(ax1, bx1), max(ay1, by1)\n    ix2, iy2 = min(ax2, bx2), min(ay2, by2)\n    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)\n    inter = iw * ih\n    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)\n    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)\n    m = min(area_a, area_b)\n    return 0.0 if m <= 0 else inter / m\n\ndef overlap_frac_of_face(face_bb, shem_bb, is_center=True):\n    fx1, fy1, fx2, fy2 = xywh_to_xyxy(face_bb, is_center=is_center)\n    sx1, sy1, sx2, sy2 = xywh_to_xyxy(shem_bb, is_center=is_center)\n    ix1, iy1 = max(fx1, sx1), max(fy1, sy1)\n    ix2, iy2 = min(fx2, sx2), min(fy2, sy2)\n    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)\n    inter = iw * ih\n    fa = max(0.0, (fx2 - fx1)) * max(0.0, (fy2 - fy1))\n    return 0.0 if fa <= 0 else inter / fa\n\ndef nms_keep_iou_or_contain(dets, iou_thr, contain_thr):\n    if not dets:\n        return []\n    dets_sorted = sorted(dets, key=lambda x: x[1], reverse=True)\n    kept = []\n    for d in dets_sorted:\n        ok = True\n        for kd in kept:\n            if (iou_xywh(d[2], kd[2], True) > iou_thr) or (inter_over_min_area_xywh(d[2], kd[2], True) > contain_thr):\n                ok = False\n                break\n        if ok:\n            kept.append(d)\n    return kept\n\ndef results_to_dets_ultralytics(r):\n    if r is None or getattr(r, \"boxes\", None) is None or len(r.boxes) == 0:\n        return []\n    b = r.boxes\n    cls  = b.cls.detach().cpu().numpy().astype(int)\n    conf = b.conf.detach().cpu().numpy().astype(float)\n    xywh = b.xywhn.detach().cpu().numpy().astype(float) if hasattr(b, \"xywhn\") and b.xywhn is not None else b.xywh.detach().cpu().numpy().astype(float)\n    dets = []\n    for c, s, bb in zip(cls, conf, xywh):\n        dets.append((int(c), float(s), (float(bb[0]), float(bb[1]), float(bb[2]), float(bb[3]))))\n    return dets\n\ndef postprocess_group_nms(r):\n    dets = results_to_dets_ultralytics(r)\n    faces = [d for d in dets if d[0] == 0]\n    shems = [d for d in dets if d[0] in (1, 2)]\n    faces = nms_keep_iou_or_contain(faces, IOU_FACE, CONTAIN_FACE)\n    shems = nms_keep_iou_or_contain(shems, IOU_GROUP_SHEM, CONTAIN_SHEM)\n    return faces + shems\n\ndef unflip_bb_xywh(bb, is_center=True):\n    x, y, w, h = bb\n    x = 1.0 - x if is_center else 1.0 - (x + w)\n    return (float(np.clip(x, 0, 1)), float(np.clip(y, 0, 1)), float(np.clip(w, 0, 1)), float(np.clip(h, 0, 1)))\n\ndef yolo_to_abs_xywh_top_left(bb_xywh_norm_center, W, H):\n    x, y, w, h = bb_xywh_norm_center\n    x1 = (x - w/2) * W\n    y1 = (y - h/2) * H\n    ww = w * W\n    hh = h * H\n    return [float(x1), float(y1), float(ww), float(hh)]\n\ndef apply_wbf_xywh(raw_dets, iou_thr, skip_thr=0.001):\n    \"\"\"\n    raw_dets: list of (score, (xc,yc,w,h)) normalized\n    returns: fused list of (score, (xc,yc,w,h)) normalized\n    \"\"\"\n    if not raw_dets:\n        return []\n    boxes, scores, labels = [], [], []\n    for s, bb in raw_dets:\n        xc, yc, w, h = bb\n        x1 = max(0.0, xc - w/2); y1 = max(0.0, yc - h/2)\n        x2 = min(1.0, xc + w/2); y2 = min(1.0, yc + h/2)\n        boxes.append([x1, y1, x2, y2])\n        scores.append(float(s))\n        labels.append(0)\n    fused_boxes, fused_scores, _ = weighted_boxes_fusion(\n        boxes_list=[boxes],\n        scores_list=[scores],\n        labels_list=[labels],\n        weights=None,\n        iou_thr=float(iou_thr),\n        skip_box_thr=float(skip_thr),\n    )\n    out = []\n    for box, sc in zip(fused_boxes, fused_scores):\n        x1, y1, x2, y2 = box\n        w = x2 - x1; h = y2 - y1\n        xc = x1 + w/2; yc = y1 + h/2\n        out.append((float(sc), (float(xc), float(yc), float(w), float(h))))\n    out.sort(key=lambda x: x[0], reverse=True)\n    return out\n\n# ==============================================================================\n# 4) COLLECT VAL IMAGES\n# ==============================================================================\nexts = (\"*.jpg\", \"*.jpeg\", \"*.png\")\nval_imgs = []\nfor e in exts:\n    val_imgs += [os.path.basename(p) for p in glob.glob(os.path.join(IMG_DIR, e))]\nval_imgs = sorted(set(val_imgs))\nprint(f\"Val images found: {len(val_imgs)}\")\nassert len(val_imgs) > 0, f\"❌ No images found in {IMG_DIR}\"\n\n# ==============================================================================\n# 5) BUILD COCO GT ONCE + gt_rp + size_cache\n# ==============================================================================\nimages = []\nannotations = []\ngt_rp = {}\nsize_cache = {}  # img_id -> (W,H)\nann_id = 1\ncategories = [{\"id\": 1, \"name\": \"Face\"}, {\"id\": 2, \"name\": \"Shemagh\"}]\n\nfor img_id, fn in enumerate(val_imgs, start=1):\n    img_path = os.path.join(IMG_DIR, fn)\n    im = cv2.imread(img_path)\n    if im is None:\n        continue\n    h_img, w_img = im.shape[:2]\n    size_cache[img_id] = (w_img, h_img)\n    images.append({\"id\": img_id, \"file_name\": fn, \"width\": w_img, \"height\": h_img})\n\n    labs = load_gt_labels(fn)\n    has_c = any(c == 1 for c, _ in labs)\n    has_w = any(c == 2 for c, _ in labs)\n    gt_rp[img_id] = 1 if (has_c and not has_w) else 0\n\n    for c, bb in labs:\n        cat = 1 if c == 0 else (2 if c in (1, 2) else None)\n        if cat is None:\n            continue\n        x1, y1, ww, hh = yolo_to_abs_xywh_top_left(bb, w_img, h_img)\n        annotations.append({\n            \"id\": ann_id, \"image_id\": img_id, \"category_id\": cat,\n            \"bbox\": [x1, y1, ww, hh],\n            \"area\": float(ww * hh),\n            \"iscrowd\": 0\n        })\n        ann_id += 1\n\ngt_coco_dict = {\"info\": {\"description\": \"val\"}, \"licenses\": [], \"images\": images, \"annotations\": annotations, \"categories\": categories}\ntmp_gt = tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".json\", delete=False)\njson.dump(gt_coco_dict, tmp_gt)\ntmp_gt.close()\n\nwith contextlib.redirect_stdout(io.StringIO()):\n    coco_gt = COCO(tmp_gt.name)\n\n# ==============================================================================\n# 6) INFER ONCE (CACHE dets_cache) WITH TTA + GROUP-NMS\n# ==============================================================================\ndets_cache = {}  # img_id -> list of (cls, score, bb_xywh_norm_center)\n\nfor img_id, fn in tqdm(list(enumerate(val_imgs, start=1)), desc=\"Infer VAL (cache dets)\"):\n    img_path = os.path.join(IMG_DIR, fn)\n    img0 = cv2.imread(img_path)\n    if img0 is None:\n        dets_cache[img_id] = []\n        continue\n\n    img_flip = cv2.flip(img0, 1) if TTA_HFLIP else None\n    img_gray = img_flip_gray = None\n\n    if TTA_GRAY:\n        g0 = cv2.cvtColor(img0, cv2.COLOR_BGR2GRAY)\n        img_gray = cv2.cvtColor(g0, cv2.COLOR_GRAY2BGR)\n        if img_flip is not None:\n            gf = cv2.cvtColor(img_flip, cv2.COLOR_BGR2GRAY)\n            img_flip_gray = cv2.cvtColor(gf, cv2.COLOR_GRAY2BGR)\n\n    dets_all = []\n\n    r0 = model.predict(img0, imgsz=IMGSZ, conf=PRED_CONF, iou=IOU_NMS, augment=False, verbose=False)[0]\n    dets_all.extend(postprocess_group_nms(r0))\n\n    if img_gray is not None:\n        rg = model.predict(img_gray, imgsz=IMGSZ, conf=PRED_CONF, iou=IOU_NMS, augment=False, verbose=False)[0]\n        dets_all.extend(postprocess_group_nms(rg))\n\n    if img_flip is not None:\n        rf = model.predict(img_flip, imgsz=IMGSZ, conf=PRED_CONF, iou=IOU_NMS, augment=False, verbose=False)[0]\n        detsf = postprocess_group_nms(rf)\n        for (c, s, bb) in detsf:\n            dets_all.append((c, s, unflip_bb_xywh(bb, True)))\n\n    if img_flip_gray is not None:\n        rfg = model.predict(img_flip_gray, imgsz=IMGSZ, conf=PRED_CONF, iou=IOU_NMS, augment=False, verbose=False)[0]\n        detsf_g = postprocess_group_nms(rfg)\n        for (c, s, bb) in detsf_g:\n            dets_all.append((c, s, unflip_bb_xywh(bb, True)))\n\n    dets_cache[img_id] = dets_all\n\n# ==============================================================================\n# 7) PRECOMPUTE FAST ARRAYS + WBF CACHES\n# ==============================================================================\nimage_ids = [img_id for img_id in sorted(dets_cache.keys()) if img_id in size_cache]\nN = len(image_ids)\nprint(\"Images used:\", N)\nassert N > 0, \"❌ No valid images used.\"\n\ngt_arr = np.array([gt_rp.get(i, 0) for i in image_ids], dtype=np.int8)\n\n# base classification scores (independent of thresholds)\nbest_c_arr = np.full(N, -1.0, dtype=np.float32)\nbest_w_arr = np.full(N, -1.0, dtype=np.float32)\nfor j, img_id in enumerate(image_ids):\n    bc = -1.0\n    bw = -1.0\n    for (c, s, _) in dets_cache[img_id]:\n        if c == 1 and s > bc: bc = s\n        elif c == 2 and s > bw: bw = s\n    best_c_arr[j] = bc\n    best_w_arr[j] = bw\n\n# WBF IOU values from grid\nface_iou_vals = sorted(set(GRID[\"WBF_FACE_IOU\"]))\nshem_iou_vals = sorted(set(GRID[\"WBF_SHEM_IOU\"]))\nface_iou_to_idx = {v: k for k, v in enumerate(face_iou_vals)}\nshem_iou_to_idx = {v: k for k, v in enumerate(shem_iou_vals)}\nF = len(face_iou_vals)\nS = len(shem_iou_vals)\n\nfaces_fused_cache = [[None]*F for _ in range(N)]\nshems_fused_cache = [[None]*S for _ in range(N)]\n\nfaces_top_score = np.full((F, N), -1.0, dtype=np.float32)\nshems_top_score = np.full((S, N), -1.0, dtype=np.float32)\nfaces_top_bb    = np.zeros((F, N, 4), dtype=np.float32)\nshems_top_bb    = np.zeros((S, N, 4), dtype=np.float32)\n\nprint(\"Precomputing WBF caches...\")\nfor j, img_id in enumerate(tqdm(image_ids, desc=\"WBF precompute\")):\n    dets_all = dets_cache[img_id]\n    faces_raw = [(s, bb) for (c, s, bb) in dets_all if c == 0]\n    shems_raw = [(s, bb) for (c, s, bb) in dets_all if c in (1, 2)]\n\n    for fi, iou_thr in enumerate(face_iou_vals):\n        fused = apply_wbf_xywh(faces_raw, iou_thr=iou_thr, skip_thr=0.001)\n        faces_fused_cache[j][fi] = fused\n        if fused:\n            faces_top_score[fi, j] = fused[0][0]\n            faces_top_bb[fi, j, :] = np.array(fused[0][1], dtype=np.float32)\n\n    for si, iou_thr in enumerate(shem_iou_vals):\n        fused = apply_wbf_xywh(shems_raw, iou_thr=iou_thr, skip_thr=0.001)\n        shems_fused_cache[j][si] = fused\n        if fused:\n            shems_top_score[si, j] = fused[0][0]\n            shems_top_bb[si, j, :] = np.array(fused[0][1], dtype=np.float32)\n\n# overlap cache between top face and top shem (per (fi,si))\noverlap_cache = np.zeros((F, S, N), dtype=np.float32)\nfor fi in range(F):\n    for si in range(S):\n        for j in range(N):\n            if faces_top_score[fi, j] >= 0 and shems_top_score[si, j] >= 0:\n                overlap_cache[fi, si, j] = overlap_frac_of_face(\n                    tuple(faces_top_bb[fi, j]), tuple(shems_top_bb[si, j]), is_center=True\n                )\n\n# ==============================================================================\n# 8) FAST F1 + UPPER-BOUND FOR ALL CONFIGS (NO COCOeval)\n# ==============================================================================\ndef fast_f1_and_ub(cfg):\n    thr_rp  = float(cfg[\"THR_RIGHT_PLACE\"])\n    ov_thr  = float(cfg[\"FACE_OVERLAP_FRAC_THR\"])\n    f_hi    = float(cfg[\"FACE_HIGH_THR\"])\n    f_min   = float(cfg[\"FACE_MIN_EXPORT\"])\n    s_hi    = float(cfg[\"SHEM_HIGH_THR\"])\n    s_min   = float(cfg[\"SHEM_MIN_EXPORT\"])\n    wf_face = float(cfg[\"WBF_FACE_IOU\"])\n    wf_shem = float(cfg[\"WBF_SHEM_IOU\"])\n\n    fi = face_iou_to_idx[wf_face]\n    si = shem_iou_to_idx[wf_shem]\n\n    # base right_place\n    pred = (best_c_arr >= thr_rp) & (best_c_arr >= best_w_arr)\n\n    # geometry override only where pred==False\n    if GEOM_OVERRIDE_ENABLED:\n        can_override = (~pred) & (faces_top_score[fi] >= FACE_MIN_FOR_OVERRIDE) & (shems_top_score[si] >= SHEM_MIN_FOR_OVERRIDE)\n        can_override &= (overlap_cache[fi, si] >= ov_thr)\n        pred = pred | can_override\n\n    # smart export existence (approx: “do we export at least 1 box?”)\n    face_exists = faces_top_score[fi] >= 0\n    shem_exists = shems_top_score[si] >= 0\n\n    # face: if >= FACE_HIGH_THR OR >= FACE_MIN_EXPORT OR rp==1 force\n    face_any = face_exists & ((faces_top_score[fi] >= f_hi) | (faces_top_score[fi] >= f_min) | pred)\n    # shem: if >= SHEM_HIGH_THR OR >= SHEM_MIN_EXPORT OR rp==1 force top-1\n    shem_any = shem_exists & ((shems_top_score[si] >= s_hi) | (shems_top_score[si] >= s_min) | pred)\n\n    if REQUIRE_BOTH_IF_RIGHT_PLACE:\n        pred = pred & face_any & shem_any\n\n    tp = np.sum((pred == 1) & (gt_arr == 1))\n    fp = np.sum((pred == 1) & (gt_arr == 0))\n    fn = np.sum((pred == 0) & (gt_arr == 1))\n\n    prec = tp / (tp + fp + 1e-9)\n    rec  = tp / (tp + fn + 1e-9)\n    f1   = 2 * prec * rec / (prec + rec + 1e-9)\n\n    # upper bound since mAP <= 1\n    ub = 0.5 * 1.0 + 0.5 * float(f1)\n    return float(f1), float(prec), float(rec), float(ub), fi, si\n\n# Build combos\nkeys = list(GRID.keys())\nvals = [GRID[k] for k in keys]\ncombos = list(itertools.product(*vals))\nprint(f\"\\nTotal configs to search: {len(combos)}\")\n\nubs = np.empty(len(combos), dtype=np.float32)\nmeta = [None] * len(combos)  # (f1, p, r, fi, si)\n\nt0 = time.time()\nfor idx, combo in enumerate(tqdm(combos, desc=\"Fast F1 upper-bounds\")):\n    cfg = dict(zip(keys, combo))\n    f1, p, r, ub, fi, si = fast_f1_and_ub(cfg)\n    ubs[idx] = ub\n    meta[idx] = (f1, p, r, fi, si)\nprint(f\"Fast stage time: {time.time()-t0:.2f}s\")\n\n# Sort by ub descending (most promising first)\norder = np.argsort(-ubs)\n\n# ==============================================================================\n# 9) EXACT mAP ONLY FOR TOP CANDIDATES (COCOeval is expensive)\n# ==============================================================================\ndef full_map_for_cfg(cfg, fi, si):\n    thr_rp = float(cfg[\"THR_RIGHT_PLACE\"])\n    ov_thr = float(cfg[\"FACE_OVERLAP_FRAC_THR\"])\n    f_hi   = float(cfg[\"FACE_HIGH_THR\"])\n    f_min  = float(cfg[\"FACE_MIN_EXPORT\"])\n    s_hi   = float(cfg[\"SHEM_HIGH_THR\"])\n    s_min  = float(cfg[\"SHEM_MIN_EXPORT\"])\n\n    preds = []\n    tp = fp = fn = 0\n\n    for j, img_id in enumerate(image_ids):\n        w_img, h_img = size_cache[img_id]\n\n        # right_place (old)\n        bc = float(best_c_arr[j]); bw = float(best_w_arr[j])\n        pred_rp = 1 if (bc >= thr_rp and bc >= bw) else 0\n        pred_rp_old = pred_rp\n\n        faces_f = faces_fused_cache[j][fi] or []\n        shems_f = shems_fused_cache[j][si] or []\n\n        # geometry override only if old==0\n        if GEOM_OVERRIDE_ENABLED and pred_rp_old == 0 and faces_f and shems_f:\n            s_face, bb_face = faces_f[0]\n            s_shem, bb_shem = shems_f[0]\n            if s_face >= FACE_MIN_FOR_OVERRIDE and s_shem >= SHEM_MIN_FOR_OVERRIDE:\n                if overlap_cache[fi, si, j] >= ov_thr:\n                    pred_rp = 1\n\n        # SMART EXPORT\n        face_exported = []\n        shem_exported = []\n\n        faces_hi = [(s, bb) for (s, bb) in faces_f if s >= f_hi]\n        if faces_hi:\n            face_exported = faces_hi[:MAX_FACE_EXPORT]\n        elif faces_f:\n            s0, bb0 = faces_f[0]\n            if (s0 >= f_min) or (pred_rp == 1):\n                face_exported = [(s0, bb0)]\n\n        shems_hi = [(s, bb) for (s, bb) in shems_f if s >= s_hi]\n        if shems_hi:\n            shem_exported = shems_hi[:MAX_SHEM_EXPORT]\n        else:\n            for i, (s0, bb0) in enumerate((shems_f or [])[:MAX_SHEM_EXPORT]):\n                if s0 >= s_min:\n                    shem_exported.append((s0, bb0))\n                elif pred_rp == 1 and i == 0:\n                    shem_exported.append((s0, bb0))  # force top-1 if rp==1\n                else:\n                    break\n\n        # Consistency enforcer\n        if REQUIRE_BOTH_IF_RIGHT_PLACE and pred_rp == 1 and not (face_exported and shem_exported):\n            pred_rp = 0\n\n        # F1 stats\n        gt = int(gt_arr[j])\n        if pred_rp == 1 and gt == 1: tp += 1\n        elif pred_rp == 1 and gt == 0: fp += 1\n        elif pred_rp == 0 and gt == 1: fn += 1\n\n        # COCO preds\n        for (s0, bb0) in face_exported:\n            preds.append({\"image_id\": img_id, \"category_id\": 1, \"bbox\": yolo_to_abs_xywh_top_left(bb0, w_img, h_img), \"score\": float(s0)})\n        for (s0, bb0) in shem_exported:\n            preds.append({\"image_id\": img_id, \"category_id\": 2, \"bbox\": yolo_to_abs_xywh_top_left(bb0, w_img, h_img), \"score\": float(s0)})\n\n    # F1 final\n    prec = tp / (tp + fp + 1e-9)\n    rec  = tp / (tp + fn + 1e-9)\n    f1   = 2 * prec * rec / (prec + rec + 1e-9)\n\n    # mAP (COCOeval) — expensive\n    if len(preds) == 0:\n        map_ = 0.0\n    else:\n        buf = io.StringIO()\n        with contextlib.redirect_stdout(buf):\n            coco_dt = coco_gt.loadRes(preds)\n            coco_eval = COCOeval(coco_gt, coco_dt, iouType=\"bbox\")\n            coco_eval.evaluate()\n            coco_eval.accumulate()\n            coco_eval.summarize()\n        map_ = float(coco_eval.stats[0])  # AP@[.50:.95]\n\n    return float(map_), float(f1), float(prec), float(rec), int(len(preds))\n\nbest = None\nbest_score = -1.0\ncoco_calls = 0\n\nt1 = time.time()\nfor rank, idx in enumerate(tqdm(order, desc=\"Exact search (budgeted)\")):\n    # optional early stop by UB\n    if STOP_IF_UB_BELOW_BEST and (float(ubs[idx]) <= best_score):\n        print(\"\\n[STOP] Upper-bound fell below best_score. Remaining configs cannot win.\")\n        break\n\n    # budget stop\n    if coco_calls >= MAX_COCO_EVAL_CALLS:\n        print(f\"\\n[STOP] Reached MAX_COCO_EVAL_CALLS={MAX_COCO_EVAL_CALLS}.\")\n        break\n\n    cfg = dict(zip(keys, combos[idx]))\n    f1_fast, p_fast, r_fast, fi, si = meta[idx]\n\n    map_, f1, p, r, n_preds = full_map_for_cfg(cfg, fi, si)\n    coco_calls += 1\n\n    score = 0.5 * map_ + 0.5 * f1\n    if score > best_score:\n        best_score = score\n        best = (score, map_, f1, p, r, n_preds, cfg)\n\n    if (coco_calls % PRINT_EVERY) == 0:\n        print(f\"  checked={coco_calls} | best_score={best_score:.6f} | current_ub={float(ubs[idx]):.6f}\")\n\nprint(f\"\\nExact stage time: {time.time()-t1:.2f}s  | COCOeval calls used: {coco_calls}\")\n\nprint(\"\\n\" + \"=\"*44)\nprint(\"🏆  BEST CONFIG FOUND (BUDGETED SEARCH)  🏆\")\nprint(\"=\"*44)\nprint(f\"Final Score : {best[0]:.6f}\")\nprint(f\"mAP@.50:.95 : {best[1]:.6f}\")\nprint(f\"F1 (RP)     : {best[2]:.6f}\")\nprint(f\"Precision   : {best[3]:.3f}\")\nprint(f\"Recall      : {best[4]:.3f}\")\nprint(f\"Total Preds : {best[5]}\")\nprint(\"-\" * 44)\nprint(\"Best Config Dictionary:\")\nprint(best[6])\nprint(\"=\"*44)\n\nBEST_CFG = best[6]  # <-- paste into your submission cell","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nmodel2 = model\nfrom tqdm import tqdm\nimport os\nimport pandas as pd\nimport numpy as np\nimport cv2\n\n# --- Ensure ensemble-boxes exists ---\ntry:\n    from ensemble_boxes import weighted_boxes_fusion\nexcept Exception:\n    !pip -q install ensemble-boxes\n    from ensemble_boxes import weighted_boxes_fusion\n\n# ----------------------------\n# Hard reset cached predictor args (kills augment=True persistence)\n# ----------------------------\ntry:\n    model2.predictor = None\nexcept Exception:\n    pass\ntry:\n    if hasattr(model2, \"overrides\"):\n        model2.overrides[\"augment\"] = False\nexcept Exception:\n    pass\n\n# ----------------------------\n# Knobs (CLEAN — defined ONCE)\n# ----------------------------\nPRED_CONF = 0.01\nIOU_NMS   = 0.70\n\n# Group NMS thresholds\nIOU_FACE         = 0.50\nIOU_GROUP_SHEM   = 0.45\nCONTAIN_FACE     = 0.95\nCONTAIN_SHEM     = 0.85\n\n# TTA\nTTA_ENABLED    = True\nTTA_IMGSZ_LIST = [640]\nTTA_HFLIP      = True\nTTA_GRAY       = True\n\n# geometry override (only if old rule gives 0)\nGEOM_OVERRIDE_ENABLED   = True\nXYWH_IS_CENTER          = True\nFACE_MIN_FOR_OVERRIDE   = 0.10\nSHEM_MIN_FOR_OVERRIDE   = 0.10\n\n# Export\nREQUIRE_BOTH_IF_RIGHT_PLACE = True\nMAX_FACE_EXPORT = 7\nMAX_SHEM_EXPORT = 7\n\n# Tiling (optional)\nTILE_ENABLED  = False\nTILE_SIZE     = 512\nTILE_OVERLAP  = 0.30\n\ncnt_tiled_used = 0\ncnt_full_used  = 0\n\noverridden_files = []\ndowngraded_files = []\n\n# ----------------------------\n# BEST VAL CONFIG (from your tuner)\n#   - If you ran the tuner cell, it may have \"best_cfg\" (dict) or \"BEST_CFG\" already.\n# ----------------------------\nif \"best_cfg\" in globals() and isinstance(globals()[\"best_cfg\"], dict):\n    BEST_CFG = globals()[\"best_cfg\"]\nelif \"BEST_CFG\" in globals() and isinstance(globals()[\"BEST_CFG\"], dict):\n    BEST_CFG = globals()[\"BEST_CFG\"]\nelse:\n    print(\"new\")\n    BEST_CFG = {\n        \"THR_RIGHT_PLACE\":       0.01,\n        \"FACE_OVERLAP_FRAC_THR\": 0.70,\n        \"FACE_HIGH_THR\":         0.10,\n        \"FACE_MIN_EXPORT\":       0.05,\n        \"SHEM_HIGH_THR\":         0.10,\n        \"SHEM_MIN_EXPORT\":       0.10,\n        \"WBF_FACE_IOU\":          0.55,\n        \"WBF_SHEM_IOU\":          0.50,\n    }\n\n# Pull thresholds out (so the rest of the code stays clean)\nTHR_RIGHT_PLACE     = float(BEST_CFG.get(\"THR_RIGHT_PLACE\", 0.01))\nFACE_OVERLAP_FRAC_THR = float(BEST_CFG.get(\"FACE_OVERLAP_FRAC_THR\", 0.70))\nFACE_HIGH_THR       = float(BEST_CFG.get(\"FACE_HIGH_THR\", 0.10))\nFACE_MIN_EXPORT     = float(BEST_CFG.get(\"FACE_MIN_EXPORT\", 0.05))\nSHEM_HIGH_THR       = float(BEST_CFG.get(\"SHEM_HIGH_THR\", 0.10))\nSHEM_MIN_EXPORT     = float(BEST_CFG.get(\"SHEM_MIN_EXPORT\", 0.10))\nWBF_FACE_IOU        = float(BEST_CFG.get(\"WBF_FACE_IOU\", 0.55))\nWBF_SHEM_IOU        = float(BEST_CFG.get(\"WBF_SHEM_IOU\", 0.50))\n\nprint(\"Applying submission inference using BEST_CFG:\")\nprint(BEST_CFG)\n\n# ----------------------------\n# Geometry / IoU helpers\n# ----------------------------\ndef _tile_coords(H, W, tile, overlap):\n    tile = int(tile)\n    step = max(1, int(tile * (1.0 - overlap)))\n\n    if H <= tile and W <= tile:\n        return [(0, 0, W, H)]\n\n    xs = list(range(0, max(1, W - tile + 1), step))\n    ys = list(range(0, max(1, H - tile + 1), step))\n\n    last_x = max(0, W - tile)\n    last_y = max(0, H - tile)\n    if xs[-1] != last_x: xs.append(last_x)\n    if ys[-1] != last_y: ys.append(last_y)\n\n    coords = []\n    for y0 in ys:\n        for x0 in xs:\n            x1 = min(W, x0 + tile)\n            y1 = min(H, y0 + tile)\n            coords.append((x0, y0, x1, y1))\n    return coords\n\ndef xywh_to_xyxy(bb, is_center=True):\n    x, y, w, h = bb\n    if is_center:\n        return (x - w/2, y - h/2, x + w/2, y + h/2)\n    else:\n        return (x, y, x + w, y + h)\n\ndef iou_xywh(bb1, bb2, is_center=True):\n    ax1, ay1, ax2, ay2 = xywh_to_xyxy(bb1, is_center=is_center)\n    bx1, by1, bx2, by2 = xywh_to_xyxy(bb2, is_center=is_center)\n    ix1, iy1 = max(ax1, bx1), max(ay1, by1)\n    ix2, iy2 = min(ax2, bx2), min(ay2, by2)\n    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)\n    inter = iw * ih\n    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)\n    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)\n    union = area_a + area_b - inter\n    return 0.0 if union <= 0 else inter / union\n\ndef inter_over_min_area_xywh(bb1, bb2, is_center=True):\n    ax1, ay1, ax2, ay2 = xywh_to_xyxy(bb1, is_center=is_center)\n    bx1, by1, bx2, by2 = xywh_to_xyxy(bb2, is_center=is_center)\n    ix1, iy1 = max(ax1, bx1), max(ay1, by1)\n    ix2, iy2 = min(ax2, bx2), min(ay2, by2)\n    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)\n    inter = iw * ih\n    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)\n    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)\n    m = min(area_a, area_b)\n    return 0.0 if m <= 0 else inter / m\n\ndef overlap_frac_of_face(face_bb, shem_bb, is_center=True):\n    fx1, fy1, fx2, fy2 = xywh_to_xyxy(face_bb, is_center=is_center)\n    sx1, sy1, sx2, sy2 = xywh_to_xyxy(shem_bb, is_center=is_center)\n    ix1, iy1 = max(fx1, sx1), max(fy1, sy1)\n    ix2, iy2 = min(fx2, sx2), min(fy2, sy2)\n    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)\n    inter = iw * ih\n    fa = max(0.0, (fx2 - fx1)) * max(0.0, (fy2 - fy1))\n    return 0.0 if fa <= 0 else inter / fa\n\n# ----------------------------\n# NMS with IoU OR containment\n# ----------------------------\ndef nms_keep_iou_or_contain(dets, iou_thr, contain_thr=0.85):\n    # dets: list (cls, score, bb_xywh_norm_center)\n    if not dets:\n        return []\n    dets_sorted = sorted(dets, key=lambda x: x[1], reverse=True)\n    kept = []\n    for d in dets_sorted:\n        ok = True\n        for kd in kept:\n            iou = iou_xywh(d[2], kd[2], is_center=XYWH_IS_CENTER)\n            com = inter_over_min_area_xywh(d[2], kd[2], is_center=XYWH_IS_CENTER)\n            if (iou > iou_thr) or (com > contain_thr):\n                ok = False\n                break\n        if ok:\n            kept.append(d)\n    return kept\n\n# ----------------------------\n# ensemble_boxes WBF (xywh center normalized) — \"what we did\"\n# ----------------------------\ndef apply_wbf_xywh(raw_dets, iou_thr, skip_thr=0.001):\n    \"\"\"\n    raw_dets: list of (score, (xc,yc,w,h)) normalized xywh-center\n    Returns fused list of (score, (xc,yc,w,h)) normalized xywh-center\n    \"\"\"\n    if not raw_dets:\n        return []\n\n    boxes, scores, labels = [], [], []\n    for s, bb in raw_dets:\n        xc, yc, w, h = bb\n        x1 = max(0.0, xc - w/2)\n        y1 = max(0.0, yc - h/2)\n        x2 = min(1.0, xc + w/2)\n        y2 = min(1.0, yc + h/2)\n        boxes.append([x1, y1, x2, y2])\n        scores.append(float(s))\n        labels.append(0)\n\n    fused_boxes, fused_scores, _ = weighted_boxes_fusion(\n        [boxes], [scores], [labels],\n        weights=None, iou_thr=float(iou_thr), skip_box_thr=float(skip_thr)\n    )\n\n    fused_raw = []\n    for box, score in zip(fused_boxes, fused_scores):\n        x1, y1, x2, y2 = box\n        w_box = x2 - x1\n        h_box = y2 - y1\n        xc_box = x1 + w_box/2\n        yc_box = y1 + h_box/2\n        fused_raw.append((float(score), (float(xc_box), float(yc_box), float(w_box), float(h_box))))\n\n    fused_raw.sort(key=lambda x: x[0], reverse=True)\n    return fused_raw\n\ndef bb_area_xywh(bb, is_center=True):\n    x1,y1,x2,y2 = xywh_to_xyxy(bb, is_center=is_center)\n    return max(0.0, x2-x1) * max(0.0, y2-y1)\n\ndef dedup_fused_wbf(fused, iou_thr, contain_thr, prefer_small=False):\n    \"\"\"\n    fused: list of (score, bb_xywh_norm_center)\n    remove nested duplicates using IoU OR containment\n    \"\"\"\n    if not fused:\n        return []\n\n    dets = [(0, float(s), bb) for (s, bb) in fused]\n\n    if prefer_small:\n        dets = sorted(dets, key=lambda d: (d[1], -bb_area_xywh(d[2], is_center=XYWH_IS_CENTER)), reverse=True)\n    else:\n        dets = sorted(dets, key=lambda d: d[1], reverse=True)\n\n    kept = []\n    for d in dets:\n        ok = True\n        for kd in kept:\n            iou = iou_xywh(d[2], kd[2], is_center=XYWH_IS_CENTER)\n            com = inter_over_min_area_xywh(d[2], kd[2], is_center=XYWH_IS_CENTER)\n            if (iou > iou_thr) or (com > contain_thr):\n                ok = False\n                break\n        if ok:\n            kept.append(d)\n\n    return [(float(s), bb) for (_, s, bb) in kept]\n\n# ----------------------------\n# Ultralytics result -> dets\n# ----------------------------\ndef results_to_dets_ultralytics(r):\n    if r is None or getattr(r, \"boxes\", None) is None or len(r.boxes) == 0:\n        return []\n    b = r.boxes\n    cls  = b.cls.detach().cpu().numpy().astype(int)\n    conf = b.conf.detach().cpu().numpy().astype(float)\n\n    if hasattr(b, \"xywhn\") and b.xywhn is not None:\n        xywh = b.xywhn.detach().cpu().numpy().astype(float)\n    else:\n        xywh = b.xywh.detach().cpu().numpy().astype(float)\n\n    dets = []\n    for c, s, bb in zip(cls, conf, xywh):\n        dets.append((int(c), float(s), (float(bb[0]), float(bb[1]), float(bb[2]), float(bb[3]))))\n    return dets\n\ndef postprocess_group_nms(r):\n    dets = results_to_dets_ultralytics(r)\n    faces = [d for d in dets if d[0] == 0]\n    shems = [d for d in dets if d[0] in (1, 2)]\n    faces = nms_keep_iou_or_contain(faces, IOU_FACE, contain_thr=CONTAIN_FACE)\n    shems = nms_keep_iou_or_contain(shems, IOU_GROUP_SHEM, contain_thr=CONTAIN_SHEM)\n    return faces + shems\n\ndef unflip_bb_xywh(bb, img_w, normalized, is_center=True):\n    x, y, w, h = bb\n    if normalized:\n        if is_center:\n            x = 1.0 - x\n        else:\n            x = 1.0 - (x + w)\n        x = float(np.clip(x, 0.0, 1.0))\n        y = float(np.clip(y, 0.0, 1.0))\n        w = float(np.clip(w, 0.0, 1.0))\n        h = float(np.clip(h, 0.0, 1.0))\n    else:\n        if is_center:\n            x = float(img_w) - x\n        else:\n            x = float(img_w) - (x + w)\n        x = float(np.clip(x, 0.0, float(img_w)))\n    return (x, y, w, h)\n\ndef predict_full_or_tiled(img, imgsz, conf, iou):\n    \"\"\"\n    Returns dets in FULL-image normalized xywh (center):\n      [(cls, score, (x,y,w,h)), ...]\n    \"\"\"\n    global cnt_tiled_used, cnt_full_used\n    H, W = img.shape[:2]\n\n    if (not TILE_ENABLED) or (H <= TILE_SIZE and W <= TILE_SIZE):\n        cnt_full_used += 1\n        r = model2.predict(img, imgsz=imgsz, conf=conf, iou=iou, augment=False, verbose=False)[0]\n        return postprocess_group_nms(r)\n\n    cnt_tiled_used += 1\n    dets = []\n    for (x0, y0, x1, y1) in _tile_coords(H, W, TILE_SIZE, TILE_OVERLAP):\n        tile_img = img[y0:y1, x0:x1]\n        th, tw = tile_img.shape[:2]\n\n        r = model2.predict(tile_img, imgsz=imgsz, conf=conf, iou=iou, augment=False, verbose=False)[0]\n        dets_tile = postprocess_group_nms(r)\n\n        for (c, s, (x, y, w, h)) in dets_tile:\n            x_full = (x * tw + x0) / W\n            y_full = (y * th + y0) / H\n            w_full = (w * tw) / W\n            h_full = (h * th) / H\n            dets.append((int(c), float(s), (float(np.clip(x_full,0,1)), float(np.clip(y_full,0,1)), float(np.clip(w_full,0,1)), float(np.clip(h_full,0,1)))))\n\n    # dedup across tiles before WBF\n    faces = [d for d in dets if d[0] == 0]\n    shems = [d for d in dets if d[0] in (1, 2)]\n    faces = nms_keep_iou_or_contain(faces, IOU_FACE, contain_thr=CONTAIN_FACE)\n    shems = nms_keep_iou_or_contain(shems, IOU_GROUP_SHEM, contain_thr=CONTAIN_SHEM)\n    return faces + shems\n\n# ----------------------------\n# Inference + build submission\n# ----------------------------\nROOT_DIR = \"/kaggle/input/dal-shemagh-detection-challenge\"\ntest_img_dir = os.path.join(ROOT_DIR, \"images/test\")\ntest_images = sorted([f for f in os.listdir(test_img_dir) if f.lower().endswith(('.jpg', '.jpeg', '.png'))])\n\nsubmission_rows = []\ncnt_old_1 = cnt_old_0 = cnt_overridden = cnt_with_dets = cnt_rp_downgraded_missing_pair = 0\n\nimgsz_list = TTA_IMGSZ_LIST if TTA_ENABLED else [TTA_IMGSZ_LIST[0]]\n\nfor fn in tqdm(test_images, desc=\"Inference (submission_safe_v3)\"):\n    img_path = os.path.join(test_img_dir, fn)\n    img0 = cv2.imread(img_path)\n    if img0 is None:\n        submission_rows.append({\"filename\": fn, \"right_place\": 0, \"prediction_string\": \"-\"})\n        continue\n\n    H, W = img0.shape[:2]\n    img_flip = cv2.flip(img0, 1) if (TTA_ENABLED and TTA_HFLIP) else None\n\n    img_gray = None\n    img_flip_gray = None\n    if TTA_ENABLED and TTA_GRAY:\n        g0 = cv2.cvtColor(img0, cv2.COLOR_BGR2GRAY)\n        img_gray = cv2.cvtColor(g0, cv2.COLOR_GRAY2BGR)\n        if img_flip is not None:\n            gf = cv2.cvtColor(img_flip, cv2.COLOR_BGR2GRAY)\n            img_flip_gray = cv2.cvtColor(gf, cv2.COLOR_GRAY2BGR)\n\n    dets_all = []  # (cls, score, bb_xywh)\n\n    for imgsz in imgsz_list:\n        # orig\n        dets_all.extend(predict_full_or_tiled(img0, imgsz=imgsz, conf=PRED_CONF, iou=IOU_NMS))\n\n        # gray\n        if img_gray is not None:\n            dets_all.extend(predict_full_or_tiled(img_gray, imgsz=imgsz, conf=PRED_CONF, iou=IOU_NMS))\n\n        # hflip\n        if img_flip is not None:\n            detsf = predict_full_or_tiled(img_flip, imgsz=imgsz, conf=PRED_CONF, iou=IOU_NMS)\n            for (c, s, bb) in detsf:\n                bb_u = unflip_bb_xywh(bb, img_w=W, normalized=True, is_center=XYWH_IS_CENTER)\n                dets_all.append((c, s, bb_u))\n\n        # hflip+gray\n        if img_flip_gray is not None:\n            detsf_g = predict_full_or_tiled(img_flip_gray, imgsz=imgsz, conf=PRED_CONF, iou=IOU_NMS)\n            for (c, s, bb) in detsf_g:\n                bb_u = unflip_bb_xywh(bb, img_w=W, normalized=True, is_center=XYWH_IS_CENTER)\n                dets_all.append((c, s, bb_u))\n\n    # ----------------------------\n    # right_place (raw dets_all => still has class 1 vs 2)\n    # ----------------------------\n    best_c = max([s for (c, s, _) in dets_all if c == 1], default=-1.0)\n    best_w = max([s for (c, s, _) in dets_all if c == 2], default=-1.0)\n    is_right_place_old = 1 if (best_c >= THR_RIGHT_PLACE and best_c >= best_w) else 0\n    cnt_old_1 += int(is_right_place_old == 1)\n    cnt_old_0 += int(is_right_place_old == 0)\n    is_right_place = is_right_place_old\n\n    # ----------------------------\n    # Merge across TTA with ensemble_boxes WBF (Face + Shemagh-group)\n    # ----------------------------\n    faces_raw = [(s, bb) for (c, s, bb) in dets_all if c == 0]\n    shems_raw = [(s, bb) for (c, s, bb) in dets_all if c in (1, 2)]\n\n    faces_fused = apply_wbf_xywh(faces_raw, iou_thr=WBF_FACE_IOU, skip_thr=0.001)\n    shems_fused = apply_wbf_xywh(shems_raw, iou_thr=WBF_SHEM_IOU, skip_thr=0.001)\n\n    # optional dedup after WBF (keeps your previous behavior)\n    faces_fused = dedup_fused_wbf(faces_fused, iou_thr=0.30, contain_thr=0.85)\n    shems_fused = dedup_fused_wbf(shems_fused, iou_thr=0.20, contain_thr=0.75, prefer_small=False)\n\n    # geometry override only if old=0\n    if GEOM_OVERRIDE_ENABLED and is_right_place_old == 0 and faces_fused and shems_fused:\n        s_face, bb_face = faces_fused[0]\n        s_shem, bb_shem = shems_fused[0]\n        if s_face >= FACE_MIN_FOR_OVERRIDE and s_shem >= SHEM_MIN_FOR_OVERRIDE:\n            frac = overlap_frac_of_face(bb_face, bb_shem, is_center=True)\n            if frac >= FACE_OVERLAP_FRAC_THR:\n                is_right_place = 1\n                cnt_overridden += 1\n                overridden_files.append(fn)\n\n    # ----------------------------\n    # EXPORT (Face class=0, Shemagh class=1)\n    #   + SMART FORCE-EXPORT protection when is_right_place==1\n    # ----------------------------\n    pred_parts = []\n    face_exported_any = False\n    shem_exported_any = False\n\n    # Face: export all >= FACE_HIGH_THR (cap), else top-1 if >= FACE_MIN_EXPORT OR rp==1\n    faces_hi = [(s, bb) for (s, bb) in faces_fused if s >= FACE_HIGH_THR]\n    if faces_hi:\n        for (s, bb) in faces_hi[:MAX_FACE_EXPORT]:\n            x, y, w, h = bb\n            pred_parts += [\"0\", f\"{s:.4f}\", f\"{x:.4f}\", f\"{y:.4f}\", f\"{w:.4f}\", f\"{h:.4f}\"]\n            face_exported_any = True\n    elif faces_fused:\n        s, bb = faces_fused[0]\n        if (s >= FACE_MIN_EXPORT) or (is_right_place == 1):\n            x, y, w, h = bb\n            pred_parts += [\"0\", f\"{s:.4f}\", f\"{x:.4f}\", f\"{y:.4f}\", f\"{w:.4f}\", f\"{h:.4f}\"]\n            face_exported_any = True\n\n    # Shemagh: export all >= SHEM_HIGH_THR (cap),\n    # else export >= SHEM_MIN_EXPORT, BUT if rp==1 force top-1 even if below min\n    shems_hi = [(s, bb) for (s, bb) in shems_fused if s >= SHEM_HIGH_THR]\n    if shems_hi:\n        for (s, bb) in shems_hi[:MAX_SHEM_EXPORT]:\n            x, y, w, h = bb\n            pred_parts += [\"1\", f\"{s:.4f}\", f\"{x:.4f}\", f\"{y:.4f}\", f\"{w:.4f}\", f\"{h:.4f}\"]\n            shem_exported_any = True\n    else:\n        for i, (s, bb) in enumerate(shems_fused[:MAX_SHEM_EXPORT]):\n            if s >= SHEM_MIN_EXPORT:\n                x, y, w, h = bb\n                pred_parts += [\"1\", f\"{s:.4f}\", f\"{x:.4f}\", f\"{y:.4f}\", f\"{w:.4f}\", f\"{h:.4f}\"]\n                shem_exported_any = True\n            elif is_right_place == 1 and i == 0:\n                # SMART FORCE: only top-1 to avoid spamming FPs\n                x, y, w, h = bb\n                pred_parts += [\"1\", f\"{s:.4f}\", f\"{x:.4f}\", f\"{y:.4f}\", f\"{w:.4f}\", f\"{h:.4f}\"]\n                shem_exported_any = True\n            else:\n                break\n\n    # If right_place==1 require BOTH exported\n    if REQUIRE_BOTH_IF_RIGHT_PLACE and is_right_place == 1:\n        if not (face_exported_any and shem_exported_any):\n            is_right_place = 0\n            cnt_rp_downgraded_missing_pair += 1\n            downgraded_files.append(fn)\n\n    pred_str = \" \".join(pred_parts) if pred_parts else \"-\"\n    if pred_str != \"-\":\n        cnt_with_dets += 1\n\n    submission_rows.append({\"filename\": fn, \"right_place\": int(is_right_place), \"prediction_string\": pred_str})\n\nsubmission_safe = pd.DataFrame(submission_rows)\nsubmission_safe.to_csv(\"submission.csv\", index=False)\n\npd.DataFrame({\"filename\": overridden_files}).to_csv(\"overridden_files_v3.csv\", index=False)\npd.DataFrame({\"filename\": downgraded_files}).to_csv(\"downgraded_files_v3.csv\", index=False)\n\nprint(\"\\nSaved: submission_safe_v3.csv\")\nprint(\"Right_place predicted:\", int(submission_safe[\"right_place\"].sum()))\nprint(\"With detections:\", int((submission_safe[\"prediction_string\"] != \"-\").sum()))\nprint(\"\\n--- right_place stats ---\")\nprint(\"Old rule -> 1:\", cnt_old_1)\nprint(\"Old rule -> 0:\", cnt_old_0)\nprint(\"Overridden (0->1):\", cnt_overridden)\nprint(\"Downgraded (1->0) missing face/shem in pred_string:\", cnt_rp_downgraded_missing_pair)\nprint(\"\\n--- TTA stats ---\")\nprint(\"TTA enabled:\", TTA_ENABLED, \"| imgsz:\", imgsz_list, \"| hflip:\", TTA_HFLIP, \"| gray:\", TTA_GRAY)\nprint(\"\\n--- TILE stats ---\")\nprint(\"TILE enabled:\", TILE_ENABLED, \"| tiled used:\", cnt_tiled_used, \"| full used:\", cnt_full_used)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T05:29:04.596162Z","iopub.execute_input":"2026-02-20T05:29:04.596539Z","iopub.status.idle":"2026-02-20T05:33:02.481630Z","shell.execute_reply.started":"2026-02-20T05:29:04.596509Z","shell.execute_reply":"2026-02-20T05:33:02.481017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}