{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":129750,"databundleVersionId":15563344,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Installing libraries\n!uv pip install -q ultralytics --system\n!uv pip install ultralytics albumentations timm scikit-learn pyyaml -q --system\n!uv pip install ensemble-boxes numpy pandas --system","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T14:13:34.259531Z","iopub.execute_input":"2026-02-17T14:13:34.259723Z","iopub.status.idle":"2026-02-17T14:13:36.136190Z","shell.execute_reply.started":"2026-02-17T14:13:34.259702Z","shell.execute_reply":"2026-02-17T14:13:36.135277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Imports\nimport os\nimport pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom ensemble_boxes import weighted_boxes_fusion\nimport albumentations as A\n\nfrom ultralytics import YOLO, RTDETR\nimport yaml\nfrom tqdm.notebook import tqdm\n\n\nimport gc\nimport torch\nimport torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T14:16:10.195125Z","iopub.execute_input":"2026-02-17T14:16:10.195557Z","iopub.status.idle":"2026-02-17T14:16:10.199912Z","shell.execute_reply.started":"2026-02-17T14:16:10.195526Z","shell.execute_reply":"2026-02-17T14:16:10.199154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Yolo data formatting\ntorch.use_deterministic_algorithms(False)\n\nDATA_ROOT = \"/kaggle/input/dal-shemagh-detection-challenge\"\n\ndataset_config = {\n    \"path\": DATA_ROOT,\n    \"train\": \"images/train\",\n    \"val\": \"images/train\",\n    \"names\": {0: \"person\", 1: \"shemagh\"},\n}\n\nwith open(\"data.yaml\", \"w\") as f:\n    yaml.dump(dataset_config, f)\n\nprint(\"data.yaml created successfully!\")\n\n\nmy_transform = A.Compose(\n    [\n        A.ToGray(p=0.2),\n        A.ChannelShuffle(p=0.2),\n        A.HorizontalFlip(p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=0.15, p=0.1),\n        A.Rotate(limit=10),\n        A.GaussianBlur(p=0.1),\n        A.Perspective(p=0.1),\n    ]\n)\n\n\nmodels_ensemble = {\n    \"yolo26x.pt\": {\"epochs\": 45, \"optimizer\": \"SGD\"},\n    \"yolo11x.pt\": {\"epochs\": 35, \"multi_scale\": True, \"optimizer\": \"SGD\"},\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T14:16:10.930931Z","iopub.execute_input":"2026-02-17T14:16:10.931712Z","iopub.status.idle":"2026-02-17T14:16:10.937571Z","shell.execute_reply.started":"2026-02-17T14:16:10.931679Z","shell.execute_reply":"2026-02-17T14:16:10.936741Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# RT-DETR training\n\nmodel = RTDETR(\"rtdetr-l.pt\")\n\n\nmodel.train(\n    data=\"data.yaml\",\n    epochs=55,\n    batch=16,\n    project=\"dal_challenge\",\n    name=\"rtdetr-l\",\n    exist_ok=True,\n    augmentations=my_transform,\n    workers=6,\n    mosaic=1.0,\n    close_mosaic=7,\n    mixup=0.05,\n    cutmix=0.1,\n    warmup_epochs=3,\n    warmup_momentum=1,\n    augment=True,\n    scale=0.25,\n    conf=0.1,\n    iou=0.3,\n    patience=10,\n    freeze=7,\n    lr0=0.001,\n    lrf=0.001,\n    weight_decay=0.0003,\n    cos_lr=True,\n    device=\"cuda\"\n)\n\n\ntorch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T14:13:50.618317Z","iopub.execute_input":"2026-02-17T14:13:50.618644Z","iopub.status.idle":"2026-02-17T14:13:54.540722Z","shell.execute_reply.started":"2026-02-17T14:13:50.618616Z","shell.execute_reply":"2026-02-17T14:13:54.539554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# YOLO11 & YOLO26 Training\nfor model_name, params in models_ensemble.items():\n    print(f\"Training {model_name} with params: {params}\")\n    model = YOLO(model_name)\n\n    model.train(\n        data=\"data.yaml\",\n        epochs=params.get(\"epochs\", 30),\n        batch=16,\n        project=\"dal_challenge\",\n        optimizer=params.get(\"optimizer\", \"adamW\"),\n        name=model_name.split(\".\")[0],\n        exist_ok=True,\n        augmentations=my_transform,\n        workers=4,\n        mosaic=1.0,\n        close_mosaic=7,\n        mixup=0.05,\n        cutmix=0.1,\n        warmup_epochs=3,\n        warmup_momentum=1,\n        augment=True\n        if model_name != \"yolo11x.pt\"\n        else False,  # Disable augmentations for multi-scale model\n        scale=0.25,\n        device=\"cuda\",\n        conf=0.1,\n        iou=0.3,\n        freeze=7,\n        multi_scale=0.3 if params.get(\"multi_scale\", False) else False,\n        lr0=0.001,\n        lrf=0.001,\n        weight_decay=0.0003,\n        cos_lr=True,\n    )\n\n    torch.cuda.empty_cache()\n    gc.collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T10:39:00.504614Z","iopub.execute_input":"2026-02-17T10:39:00.505368Z","iopub.status.idle":"2026-02-17T10:39:09.785459Z","shell.execute_reply.started":"2026-02-17T10:39:00.505334Z","shell.execute_reply":"2026-02-17T10:39:09.783051Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"import csv, os\nfrom pathlib import Path\nfrom PIL import Image\n\n# Load trained models\nmodel_paths = {\n    'yolo26x': '/kaggle/working/runs/detect/dal_challenge/yolo26x/weights/best.pt',\n    'yolo11x': '/kaggle/working/runs/detect/dal_challenge/yolo11x/weights/best.pt',\n    'rtdetr-l': '/kaggle/working/runs/detect/dal_challenge/rtdetr-l/weights/best.pt'\n}\n\nmodels = []\nfor name, path in model_paths.items():\n    if os.path.exists(path):\n        if 'rtdetr' in name:\n            m = RTDETR(path)\n        else:\n            m = YOLO(path)\n        models.append((name, m))\n        print(f\"Loaded {name} from {path}\")\n    else:\n        print(f\"Model not found: {path}\")\n\nprint(f\"\\nTotal models loaded: {len(models)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T10:39:18.605016Z","iopub.execute_input":"2026-02-17T10:39:18.605401Z","iopub.status.idle":"2026-02-17T10:39:18.809668Z","shell.execute_reply.started":"2026-02-17T10:39:18.605359Z","shell.execute_reply":"2026-02-17T10:39:18.808968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Run inference on all models and collect predictions for WBF\n\ndef filter_invalid_boxes(boxes, scores, labels, conf_thr=-1.0):\n    \"\"\"Keep only non-degenerate boxes\"\"\"\n    filtered_boxes, filtered_scores, filtered_labels = [], [], []\n    for b, s, l in zip(boxes, scores, labels):\n        if abs(b[2] - b[0]) <= 1e-6 or abs(b[3] - b[1]) <= 1e-6:\n            continue\n        if conf_thr > 0 and s < conf_thr:\n            continue\n        filtered_boxes.append(b)\n        filtered_scores.append(s)\n        filtered_labels.append(l)\n    return filtered_boxes, filtered_scores, filtered_labels\n\ndef run_inference_all_models(models, test_images_path, imgsz=640, pre_conf=1e-3, iou_thr=0.45, max_det=50):\n    \"\"\"\n    Run inference on all models with very low confidence threshold\n    to collect all candidate boxes for WBF fusion.\n    \"\"\"\n    image_paths = sorted(Path(test_images_path).glob(\"*.jpg\"))\n    predictions = {}\n    \n    for model_idx, (model_name, model) in enumerate(models):\n        print(f\"\\nRunning inference with {model_name}...\")\n        predictions[model_name] = {}\n        \n        for img_path in tqdm(image_paths, desc=model_name):\n            image_id = img_path.name\n            img = Image.open(img_path).convert(\"RGB\")\n            w, h = img.size\n            \n            # Run prediction with very low conf to keep all candidates\n            results = model.predict(\n                source=str(img_path),\n                conf=pre_conf,\n                iou=iou_thr,\n                max_det=max_det,\n                augment=True if model_name != \"yolo26x\" else False,\n                imgsz=imgsz,\n                verbose=False\n            )\n            res = results[0]\n            \n            raw_boxes = res.boxes.xyxy.cpu().numpy().tolist()\n            raw_scores = res.boxes.conf.cpu().numpy().tolist()\n            raw_labels = res.boxes.cls.cpu().numpy().tolist()\n            \n            # Normalize to [0,1] and filter invalid geometry\n            rel_boxes = [[x1/w, y1/h, x2/w, y2/h] for x1, y1, x2, y2 in raw_boxes]\n            boxes, scores, labels = filter_invalid_boxes(rel_boxes, raw_scores, raw_labels)\n            \n            predictions[model_name][image_id] = {\n                \"boxes\": boxes,\n                \"scores\": scores,\n                \"labels\": labels\n            }\n    \n    return predictions, [p.name for p in image_paths]\n\ntest_images_path = f'{DATA_ROOT}/images/test'\npredictions, image_ids = run_inference_all_models(\n    models, \n    test_images_path, \n    imgsz=640,\n    pre_conf=0.1,\n    iou_thr=0.35,\n    # max_det=50\n)\n\nprint(f\"\\nCollected predictions for {len(image_ids)} images from {len(models)} models\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T10:42:13.914838Z","iopub.execute_input":"2026-02-17T10:42:13.915191Z","iopub.status.idle":"2026-02-17T10:42:16.762512Z","shell.execute_reply.started":"2026-02-17T10:42:13.915160Z","shell.execute_reply":"2026-02-17T10:42:16.761447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying WBF\n\ndef apply_wbf(predictions, image_ids, iou_thr=0.35, skip_box_thr=0.0, conf_post=None, conf_type=\"avg\"):\n    # Handle conf_post as dict or scalar\n    if conf_post is None:\n        conf_post = {0: 0.3, 1: 0.3}  # default\n    elif isinstance(conf_post, (int, float)):\n        conf_post = {0: conf_post, 1: conf_post}\n    \n    submission_data = []\n    \n    for image_id in tqdm(image_ids, desc=\"Applying WBF\"):\n        all_boxes, all_scores, all_labels = [], [], []\n        \n        for model_name, model_preds in predictions.items():\n            pred = model_preds.get(image_id)\n            if not pred or not pred[\"boxes\"]:\n                continue\n            all_boxes.append(pred[\"boxes\"])\n            all_scores.append(pred[\"scores\"])\n            all_labels.append(pred[\"labels\"])\n        \n        if not all_boxes:\n            prediction_str = \"-\"\n        else:\n            # Apply WBF\n            fused_boxes, fused_scores, fused_labels = weighted_boxes_fusion(\n                all_boxes,\n                all_scores,\n                all_labels,\n                iou_thr=iou_thr,\n                skip_box_thr=skip_box_thr,\n                conf_type=conf_type,\n                weights=[1.5, 1, 2] if len(all_boxes) == 3 else None\n            )\n            \n            # Filter by per-class confidence threshold AFTER fusion\n            keep = []\n            for i, (s, lbl) in enumerate(zip(fused_scores, fused_labels)):\n                cls_id = int(lbl)\n                thr = conf_post.get(cls_id, 0.15)  # default 0.15 if class not in dict\n                if s >= thr:\n                    keep.append(i)\n            \n            if not keep:\n                prediction_str = \"-\"\n            else:\n                # Convert from xyxy to xywh (center format) for submission\n                pred_parts = []\n                for i in keep:\n                    x1, y1, x2, y2 = fused_boxes[i]\n                    cx = (x1 + x2) / 2\n                    cy = (y1 + y2) / 2\n                    w = x2 - x1\n                    h = y2 - y1\n                    cls = int(fused_labels[i])\n                    conf = fused_scores[i]\n                    pred_parts.append(f\"{cls} {conf:.4f} {cx:.4f} {cy:.4f} {w:.4f} {h:.4f}\")\n                prediction_str = \" \".join(pred_parts)\n        \n        submission_data.append([image_id, prediction_str])\n    \n    return submission_data\n\n# Apply WBF with per-class confidence thresholds\nwbf_submission = apply_wbf(\n    predictions, \n    image_ids,\n    iou_thr=0.35,\n    skip_box_thr=0.001,\n    conf_post={0: 0.34, 1: 0.24},  # Per-class: {person: 0.3, shemagh: 0.3}\n    conf_type=\"max\"\n)\n\ndf_wbf = pd.DataFrame(wbf_submission, columns=['filename', 'prediction_string'])\nprint(f\"\\nWBF submission shape: {df_wbf.shape}\")\nprint(df_wbf.head())\n\n# Count detections\ndef count_detections(pred_str):\n    if pred_str == '-':\n        return 0, 0\n    parts = pred_str.split()\n    persons = sum(1 for i in range(0, len(parts), 6) if int(parts[i]) == 0)\n    shemaghs = sum(1 for i in range(0, len(parts), 6) if int(parts[i]) == 1)\n    return persons, shemaghs\n\ntotal_persons = sum(count_detections(p)[0] for p in df_wbf['prediction_string'])\ntotal_shemaghs = sum(count_detections(p)[1] for p in df_wbf['prediction_string'])\nprint(f\"\\nTotal detections - Persons: {total_persons}, Shemaghs: {total_shemaghs}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T14:14:00.752012Z","iopub.execute_input":"2026-02-17T14:14:00.752853Z","iopub.status.idle":"2026-02-17T14:14:02.014522Z","shell.execute_reply.started":"2026-02-17T14:14:00.752819Z","shell.execute_reply":"2026-02-17T14:14:02.013579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_wbf.to_csv(\"submission_wbf.csv\", index=False)\ndf_wbf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T10:39:09.801425Z","iopub.status.idle":"2026-02-17T10:39:09.801893Z","shell.execute_reply.started":"2026-02-17T10:39:09.801635Z","shell.execute_reply":"2026-02-17T10:39:09.801687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compute right_place using IoU-based heuristic\n\ndef parse_prediction_string(pred_str):\n    \"\"\"Parse prediction string to get person and shemagh boxes\"\"\"\n    if pred_str == '-' or pd.isna(pred_str):\n        return [], []\n    \n    person_boxes, shemagh_boxes = [], []\n    parts = pred_str.strip().split(' ')\n    \n    for i in range(0, len(parts), 6):\n        if i + 5 < len(parts):\n            cls = int(parts[i])\n            conf = float(parts[i+1])\n            x, y, w, h = float(parts[i+2]), float(parts[i+3]), float(parts[i+4]), float(parts[i+5])\n            if cls == 0:\n                person_boxes.append([x, y, w, h, conf])\n            elif cls == 1:\n                shemagh_boxes.append([x, y, w, h, conf])\n    \n    return person_boxes, shemagh_boxes\n\ndef calculate_iou_xywh(box1, box2):\n    \"\"\"Calculate IoU between two boxes [x, y, w, h] in center format\"\"\"\n    b1_x1, b1_x2 = box1[0] - box1[2]/2, box1[0] + box1[2]/2\n    b1_y1, b1_y2 = box1[1] - box1[3]/2, box1[1] + box1[3]/2\n    b2_x1, b2_x2 = box2[0] - box2[2]/2, box2[0] + box2[2]/2\n    b2_y1, b2_y2 = box2[1] - box2[3]/2, box2[1] + box2[3]/2\n\n    inter_x1 = max(b1_x1, b2_x1)\n    inter_y1 = max(b1_y1, b2_y1)\n    inter_x2 = min(b1_x2, b2_x2)\n    inter_y2 = min(b1_y2, b2_y2)\n\n    inter_area = max(0, inter_x2 - inter_x1) * max(0, inter_y2 - inter_y1)\n    b1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1)\n    b2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1)\n    union_area = b1_area + b2_area - inter_area\n    return inter_area / union_area if union_area > 0 else 0\n\ndef compute_right_place(pred_str, iou_threshold=0.10, min_area_ratio=0.4):\n    \"\"\"Compute right_place based on IoU between person head and shemagh\"\"\"\n    person_boxes, shemagh_boxes = parse_prediction_string(pred_str)\n    right_place = 0\n    if person_boxes and shemagh_boxes:\n        best_iou = 0.0\n        best_pair = None\n\n        # 1. Ищем наилучшую пару (человек, платок) по IoU\n        for p_idx, p_box in enumerate(person_boxes):\n            for s_idx, s_box in enumerate(shemagh_boxes):\n                iou = calculate_iou_xywh(p_box, s_box)  # порядок не важен\n                if iou > best_iou:\n                    best_iou = iou\n                    best_pair = (p_idx, s_idx)\n\n        if best_pair is not None:\n            p_box = person_boxes[best_pair[0]]\n            s_box = shemagh_boxes[best_pair[1]]\n\n            # 2. Критерии правильного размещения\n            iou_threshold = 0.01        # можно подобрать (0.07..0.15)\n            vertical_threshold = 0.0    # центр платка выше центра человека (y↑ вниз, значит s_y < p_y)\n            min_area_ratio = 0.4\n\n            iou_ok = best_iou > iou_threshold\n            vertical_ok = s_box[1] < p_box[1] - vertical_threshold\n            area_ratio = (s_box[2] * s_box[3]) / (p_box[2] * p_box[3])\n            size_ok = area_ratio > min_area_ratio\n\n            # Итоговое решение: IoU + вертикальное положение (можно добавить size_ok при необходимости)\n            if iou_ok and size_ok:   # и, опционально, size_ok\n                right_place = 1\n    \n    return right_place\n\n# Apply right_place computation to WBF predictions\ndf_wbf['right_place'] = df_wbf['prediction_string'].apply(compute_right_place)\n\nprint(f\"Right place distribution:\")\nprint(df_wbf['right_place'].value_counts())\n\n# Save initial WBF submission\ndf_wbf.to_csv('submission.csv', index=False)\nprint(f\"\\nSaved WBF submission to submission_wbf.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T10:42:49.347239Z","iopub.execute_input":"2026-02-17T10:42:49.347781Z","iopub.status.idle":"2026-02-17T10:42:49.361432Z","shell.execute_reply.started":"2026-02-17T10:42:49.347749Z","shell.execute_reply":"2026-02-17T10:42:49.360657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use existing submission file for pseudo-labels (no YOLO inference needed)\nfrom collections import defaultdict\n\n# Load ground truth training labels\ntrain_labels_df = pd.read_csv(f'/kaggle/input/dal-shemagh-detection-challenge/train_labels.csv')\ntrain_labels_df['right_place'] = train_labels_df['right_place'].map({'TRUE': 1, 'FALSE': 0, True: 1, False: 0})\n\n# Load pre-generated submission with YOLO detections (test images)\nsubmission_df = pd.read_csv('submission.csv')\n\ndef parse_prediction_string(pred_str):\n    \"\"\"Parse YOLO prediction string to check for person/shemagh\"\"\"\n    if pred_str == '-' or pd.isna(pred_str):\n        return False, False\n    \n    has_person = False\n    has_shemagh = False\n    parts = pred_str.strip().split(' ')\n    \n    for i in range(0, len(parts), 6):\n        if i + 5 < len(parts):\n            cls = int(parts[i])\n            if cls == 0:\n                has_person = True\n            elif cls == 1:\n                has_shemagh = True\n    \n    return has_person, has_shemagh\n\ndef parse_label_file(label_path):\n    \"\"\"Parse YOLO label file to check for person/shemagh\"\"\"\n    has_person = False\n    has_shemagh = False\n    \n    if os.path.exists(label_path):\n        with open(label_path, 'r') as f:\n            for line in f:\n                parts = line.strip().split()\n                if len(parts) >= 5:\n                    cls = int(parts[0])\n                    if cls == 0:\n                        has_person = True\n                    elif cls == 1:\n                        has_shemagh = True\n    \n    return has_person, has_shemagh\n\n# Build pseudo-labels from test submission file\ntrain_pseudo_data = []\nfor idx, row in submission_df.iterrows():\n    filename = row['filename']\n    pred_str = row['prediction_string']\n    \n    has_person, has_shemagh = parse_prediction_string(pred_str)\n    right_place = row['right_place']\n    \n    train_pseudo_data.append({\n        'filename': filename,\n        'has_person': has_person,\n        'has_shemagh': has_shemagh,\n        'right_place': right_place,\n        'source': 'test'\n    })\n\n# Add training data with ground truth labels - READ from label files!\nlabels_dir = os.path.join(DATA_ROOT, 'labels/train')\nfor idx, row in train_labels_df.iterrows():\n    filename = row['filename']\n    right_place = row['right_place']\n    \n    # Read the actual label file to determine has_person/has_shemagh\n    label_filename = filename.replace('.jpg', '.txt').replace('.png', '.txt')\n    label_path = os.path.join(labels_dir, label_filename)\n    \n    has_person, has_shemagh = parse_label_file(label_path)\n    \n    train_pseudo_data.append({\n        'filename': filename,\n        'has_person': has_person,\n        'has_shemagh': has_shemagh,\n        'right_place': right_place,\n        'source': 'train'\n    })\n\ndf_pseudo = pd.DataFrame(train_pseudo_data)\nprint(f\"Loaded pseudo-labels: {len(submission_df)} from test, {len(train_labels_df)} from train\")\nprint(f\"Total: {len(df_pseudo)} images\")\nprint(f\"Right place distribution: {df_pseudo['right_place'].value_counts().to_dict()}\")\n\n# Show breakdown of train data categories\ntrain_data = df_pseudo[df_pseudo['source'] == 'train']\nprint(f\"\\nTrain data breakdown:\")\nprint(f\"  Person only: {((train_data['has_person']) & (~train_data['has_shemagh'])).sum()}\")\nprint(f\"  Shemagh only: {((~train_data['has_person']) & (train_data['has_shemagh'])).sum()}\")\nprint(f\"  Both: {((train_data['has_person']) & (train_data['has_shemagh'])).sum()}\")\nprint(f\"  Neither: {((~train_data['has_person']) & (~train_data['has_shemagh'])).sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T10:39:09.808745Z","iopub.status.idle":"2026-02-17T10:39:09.810247Z","shell.execute_reply.started":"2026-02-17T10:39:09.809996Z","shell.execute_reply":"2026-02-17T10:39:09.810035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create balanced dataset - equal samples from each category\nimport random\nrandom.seed(42)\n\ndef parse_boxes_from_prediction(pred_str):\n    \"\"\"Parse prediction string to get person and shemagh boxes\"\"\"\n    if pred_str == '-' or pd.isna(pred_str):\n        return [], []\n    \n    person_boxes = []\n    shemagh_boxes = []\n    parts = pred_str.strip().split(' ')\n    \n    for i in range(0, len(parts), 6):\n        if i + 5 < len(parts):\n            cls = int(parts[i])\n            conf = float(parts[i+1])\n            x, y, w, h = float(parts[i+2]), float(parts[i+3]), float(parts[i+4]), float(parts[i+5])\n            if cls == 0:\n                person_boxes.append([x, y, w, h, conf])\n            elif cls == 1:\n                shemagh_boxes.append([x, y, w, h, conf])\n    \n    return person_boxes, shemagh_boxes\n\ndef calculate_iou(box1, box2):\n    \"\"\"Calculate IoU between two boxes [x, y, w, h]\"\"\"\n    b1_x1, b1_x2 = box1[0] - box1[2]/2, box1[0] + box1[2]/2\n    b1_y1, b1_y2 = box1[1] - box1[3]/2, box1[1] + box1[3]/2\n    b2_x1, b2_x2 = box2[0] - box2[2]/2, box2[0] + box2[2]/2\n    b2_y1, b2_y2 = box2[1] - box2[3]/2, box2[1] + box2[3]/2\n\n    inter_x1 = max(b1_x1, b2_x1)\n    inter_y1 = max(b1_y1, b2_y1)\n    inter_x2 = min(b1_x2, b2_x2)\n    inter_y2 = min(b1_y2, b2_y2)\n\n    inter_area = max(0, inter_x2 - inter_x1) * max(0, inter_y2 - inter_y1)\n    b1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1)\n    b2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1)\n    union_area = b1_area + b2_area - inter_area\n    return inter_area / union_area if union_area > 0 else 0\n\n# Create lookup for test predictions\ntest_predictions = {}\nfor idx, row in submission_df.iterrows():\n    test_predictions[row['filename']] = row['prediction_string']\n\n# Categorize based on YOLO detections and ground truth right_place\ncategory_a = []  # person only (no shemagh) -> right_place=0\ncategory_b = []  # person + shemagh, wrong place (right_place=0)\ncategory_c = []  # person + shemagh, right place (right_place=1)\n\n# Stricter filtering for test data in category_b\n# Only include test images where IoU is very low (confident wrong place)\nIOU_THRESHOLD_FOR_FALSE = 0.075  # Very low IoU = confident it's not on the head\n\nfor idx, row in df_pseudo.iterrows():\n    filename = row['filename']\n    has_person = row['has_person']\n    has_shemagh = row['has_shemagh']\n    right_place = row['right_place']\n    source = row['source']\n    \n    if has_person and not has_shemagh:\n        category_a.append({'filename': filename, 'source': source})\n    elif has_person and has_shemagh:\n        if right_place == 1:\n            category_c.append({'filename': filename, 'source': source})\n        else:\n            # For category_b: always include train, filter test strictly\n            if source == 'train':\n                category_b.append({'filename': filename, 'source': source})\n            else:\n                # For test data, only include if IoU is very low (confident wrong place)\n                pred_str = test_predictions.get(filename, '-')\n                person_boxes, shemagh_boxes = parse_boxes_from_prediction(pred_str)\n                \n                if person_boxes and shemagh_boxes:\n                    # Calculate max IoU between any person-shemagh pair\n                    max_iou = 0\n                    for p_box in person_boxes:\n                        for s_box in shemagh_boxes:\n                            iou = calculate_iou(p_box[:4], s_box[:4])\n                            max_iou = max(max_iou, iou)\n                    \n                    # Only include if IoU is very low (confident it's truly wrong place) \n                    if max_iou < IOU_THRESHOLD_FOR_FALSE:\n                        if filename in [\"768.jpg\", '443.jpg', '436.jpg']:\n                            continue\n                        category_b.append({'filename': filename, 'source': source})\n\nprint(f\"Category A (person only): {len(category_a)}\")\nprint(f\"Category B (person+shemagh, wrong): {len(category_b)}\")\nprint(f\"  - from train: {sum(1 for x in category_b if x['source'] == 'train')}\")\nprint(f\"  - from test (filtered): {sum(1 for x in category_b if x['source'] == 'test')}\")\nprint(f\"Category C (person+shemagh, right): {len(category_c)}\")\n\n# Find minimum category size for balancing\nmin_size = min(len(category_a), len(category_b), len(category_c))\nprint(f\"\\nBalancing to {min_size} samples per category ({min_size * 3} total)\")\n\n# Sample equally from each category\nsampled_a = random.sample(category_a, min(len(category_a), min_size))\nsampled_b = random.sample(category_b, min(len(category_b), min_size))\nsampled_c = random.sample(category_c, min(len(category_c), min_size))\n\n# Create training data\ntrain_data = []\n\nfor item in sampled_a:\n    train_data.append({'filename': item['filename'], 'source': item['source'], 'label': 0})\nfor item in sampled_b:\n    train_data.append({'filename': item['filename'], 'source': item['source'], 'label': 0})\nfor item in sampled_c:\n    train_data.append({'filename': item['filename'], 'source': item['source'], 'label': 1})\n\ndf_convnext_train = pd.DataFrame(train_data)\ndf_convnext_train = df_convnext_train.sample(frac=1, random_state=42).reset_index(drop=True)\n\nprint(f\"\\nTraining samples: {len(df_convnext_train)}\")\nprint(f\"Label distribution:\\n{df_convnext_train['label'].value_counts()}\")\nprint(f\"Source distribution:\\n{df_convnext_train['source'].value_counts()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PyTorch Dataset and DataLoader for ConvNeXt training\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nclass ShemaghDataset(Dataset):\n    def __init__(self, df, data_root, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.data_root = data_root\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        source = row['source']\n        \n        # Use correct directory based on source\n        if source == 'train':\n            img_dir = os.path.join(self.data_root, 'images/train')\n        else:\n            img_dir = os.path.join(self.data_root, 'images/test')\n        \n        img_path = os.path.join(img_dir, row['filename'])\n        image = np.array(Image.open(img_path).convert('RGB'))\n        \n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        \n        label = torch.tensor(row['label'], dtype=torch.float32)\n        return image, label\n\n# Albumentations transforms with ToGray and ChannelShuffle\ntrain_transform = A.Compose([\n    A.Resize(224, 224),\n    A.HorizontalFlip(p=0.5),\n    A.Rotate(limit=10, p=0.5),\n    A.ToGray(p=0.2),\n    A.ChannelShuffle(p=0.24),\n    A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.3),\n    A.GaussianBlur(blur_limit=(3, 5), p=0.1),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2(),\n])\n\nval_transform = A.Compose([\n    A.Resize(224, 224),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2(),\n])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# StratifiedKFold Training with OOF predictions\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nLABEL_SMOOTHING = 0.1  # Manual label smoothing: 0 -> 0.1, 1 -> 0.9\nN_FOLDS = 5\nNUM_EPOCHS = 20\n\ndef train_epoch(model, loader, criterion, optimizer, device):\n    model.train()\n    total_loss = 0\n    all_preds, all_labels = [], []\n    \n    for images, labels in tqdm(loader, desc='Training', leave=False):\n        images, labels = images.to(device).float(), labels.to(device)\n        \n        # Apply manual label smoothing with torch.clamp\n        labels_smooth = torch.clamp(labels, min=LABEL_SMOOTHING, max=1.0 - LABEL_SMOOTHING)\n        \n        optimizer.zero_grad()\n        outputs = model(images).squeeze(-1)\n        loss = criterion(outputs, labels_smooth)\n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n        preds = (torch.sigmoid(outputs) > 0.5).float()\n        all_preds.extend(preds.detach().cpu().numpy().flatten())\n        all_labels.extend(labels.detach().cpu().numpy().flatten())\n    \n    f1 = f1_score(all_labels, all_preds)\n    acc = accuracy_score(all_labels, all_preds)\n    return total_loss / len(loader), f1, acc\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0\n    all_preds, all_labels, all_probs = [], [], []\n    \n    with torch.no_grad():\n        for images, labels in tqdm(loader, desc='Validation', leave=False):\n            images, labels = images.to(device).float(), labels.to(device)\n            outputs = model(images).squeeze(-1)\n            loss = criterion(outputs, labels)\n            \n            total_loss += loss.item()\n            probs = torch.sigmoid(outputs)\n            preds = (probs > 0.5).float()\n            all_probs.extend(probs.cpu().numpy().flatten())\n            all_preds.extend(preds.cpu().numpy().flatten())\n            all_labels.extend(labels.cpu().numpy().flatten())\n    \n    f1 = f1_score(all_labels, all_preds)\n    acc = accuracy_score(all_labels, all_preds)\n    return total_loss / len(loader), f1, acc, np.array(all_probs)\n\ndef train_fold(train_df, val_df, fold_idx, save_name):\n    \"\"\"Train a single fold and return OOF predictions\"\"\"\n    train_dataset = ShemaghDataset(train_df, DATA_ROOT, transform=train_transform)\n    val_dataset = ShemaghDataset(val_df, DATA_ROOT, transform=val_transform)\n    \n    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, pin_memory=False)\n    val_loader = DataLoader(val_dataset, batch_size=8, shuffle=False, pin_memory=False)\n    \n    # Fresh model for each fold\n    model = timm.create_model('timm/convnext_small.dinov3_lvd1689m', pretrained=True, num_classes=1)\n    model = model.to(device)\n    \n    pos_weight = torch.tensor([1.5]).to(device)\n    criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5, weight_decay=0.01)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)\n    \n    best_f1 = 0\n    best_probs = None\n    \n    for epoch in range(NUM_EPOCHS):\n        train_loss, train_f1, train_acc = train_epoch(model, train_loader, criterion, optimizer, device)\n        val_loss, val_f1, val_acc, val_probs = validate(model, val_loader, criterion, device)\n        scheduler.step()\n        \n        if val_f1 > best_f1:\n            best_f1 = val_f1\n            best_probs = val_probs\n            torch.save(model.state_dict(), save_name)\n        \n        if (epoch + 1) % 5 == 0:\n            print(f\"  Epoch {epoch+1}/{NUM_EPOCHS} - Val F1: {val_f1:.4f}, Acc: {val_acc:.4f}\")\n    \n    print(f\"  Best Val F1: {best_f1:.4f}\")\n    return best_probs, best_f1\n\n# Setup device\ndevice = torch.device(\"mps\" if torch.backends.mps.is_available() else \"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Device: {device}\")\n\n# StratifiedKFold\nskf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)\noof_preds = np.zeros(len(df_convnext_train))\nfold_scores = []\n\nprint(f\"\\n{'='*50}\")\nprint(f\"Starting {N_FOLDS}-Fold Cross-Validation\")\nprint(f\"{'='*50}\\n\")\n\nfor fold_idx, (train_idx, val_idx) in enumerate(skf.split(df_convnext_train, df_convnext_train['label'])):\n    print(f\"Fold {fold_idx + 1}/{N_FOLDS}\")\n    \n    train_fold_df = df_convnext_train.iloc[train_idx].reset_index(drop=True)\n    val_fold_df = df_convnext_train.iloc[val_idx].reset_index(drop=True)\n    \n    fold_probs, fold_f1 = train_fold(\n        train_fold_df, val_fold_df, fold_idx, \n        f'convnext_fold{fold_idx}.pth'\n    )\n    \n    oof_preds[val_idx] = fold_probs\n    fold_scores.append(fold_f1)\n    print()\n\n# OOF evaluation\noof_binary = (oof_preds > 0.5).astype(int)\noof_f1 = f1_score(df_convnext_train['label'].values, oof_binary)\noof_acc = accuracy_score(df_convnext_train['label'].values, oof_binary)\n\nprint(f\"{'='*50}\")\nprint(f\"Cross-Validation Results\")\nprint(f\"{'='*50}\")\nprint(f\"Fold F1 scores: {[f'{s:.4f}' for s in fold_scores]}\")\nprint(f\"Mean Fold F1: {np.mean(fold_scores):.4f} (+/- {np.std(fold_scores):.4f})\")\nprint(f\"OOF F1: {oof_f1:.4f}, OOF Acc: {oof_acc:.4f}\")\n\n# Save OOF predictions\ndf_convnext_train['oof_prob'] = oof_preds\ndf_convnext_train['oof_pred'] = oof_binary\ndf_convnext_train.to_csv('convnext_oof.csv', index=False)\nprint(f\"\\nOOF predictions saved to convnext_oof.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference with ConvNeXt - Ensemble of fold models + full model\nimport glob as glob_module\nimport timm\n# Load all fold models + full model for ensemble\nmodel_paths = sorted(glob_module.glob('convnext_fold*.pth'))\nmodels = []\ndevice = torch.device(\"mps\" if torch.backends.mps.is_available() else \"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nfor path in model_paths:\n    m = timm.create_model('timm/convnext_small.dinov3_lvd1689m', pretrained=False, num_classes=1)\n    m.load_state_dict(torch.load(path, map_location=device))\n    m = m.to(device)\n    m.eval()\n    models.append(m)\n    print(f\"Loaded: {path}\")\n\nprint(f\"\\nEnsembling {len(models)} models\")\n\n# Weights for ensemble: give more weight to models 3 and 4 (fold2, fold3)\n# Order: fold0, fold1, fold2, fold3, fold4, full\nmodel_weights = [1.0, 1.0, 1.5, 1.5, 1.3]\nmodel_weights = np.array(model_weights[:len(models)])  # Trim if fewer models\nmodel_weights = model_weights / model_weights.sum()  # Normalize\nprint(f\"Model weights: {model_weights}\")\n\n# Load existing submission file with YOLO predictions\nexisting_sub = pd.read_csv('submission_wbf.csv')\nsubmission_data_convnext = []\n\n\nfor idx, row in tqdm(existing_sub.iterrows(), total=len(existing_sub), desc='Ensemble Inference'):\n    filename = row['filename']\n    prediction_str = row['prediction_string']\n    \n    # Parse to check for person and shemagh\n    has_person, has_shemagh = parse_prediction_string(prediction_str)\n    \n    if prediction_str == '-' or not (has_person and has_shemagh):\n        right_place = 0\n    else:\n        # Use ensemble for right_place classification\n        img_path = os.path.join(f'{DATA_ROOT}/images/test', filename)\n        img = np.array(Image.open(img_path).convert('RGB'))\n        augmented = val_transform(image=img)\n        img_tensor = augmented['image'].unsqueeze(0).to(device).float()\n        \n        probs = []\n        with torch.no_grad():\n            for m in models:\n                output = m(img_tensor).squeeze()\n                prob = torch.sigmoid(output).item()\n                probs.append(prob)\n        \n        # Weighted average ensemble\n        right_place = np.sum(np.array(probs) * model_weights)\n    \n    submission_data_convnext.append([filename, right_place, prediction_str])\n\ndf_sub_convnext = pd.DataFrame(submission_data_convnext, columns=['filename', 'right_place', 'prediction_string'])\ndf_sub_convnext.to_csv('submission_convnext.csv', index=False)\n\nprint(\"\\nsubmission_convnext.csv created!\")\nprint(df_sub_convnext.head())\nprint(f\"\\nRight place > 0.5: {(df_sub_convnext['right_place'] > 0.5).sum()}\")\nprint(f\"Right place <= 0.5: {(df_sub_convnext['right_place'] <= 0.5).sum()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_sub_convnext['right_place_prob'] = df_sub_convnext['right_place']\ndf_sub_convnext['right_place'] = (df_sub_convnext['right_place'] > 0.5).astype(int)\ndf_sub_convnext.to_csv('submission_convnext.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter small person detections, high aspect ratio, and apply NMS\n\nMIN_AREA_RATIO = 0.025      # Minimum area ratio relative to largest person\nMAX_ASPECT_RATIO = 4.7    # Max aspect ratio (w/h or h/w) - filters elongated boxes\nNMS_IOU_THRESHOLD = 0.051   # IoU threshold for NMS\nMULTIPLE_PERSON_CONF_THRESHOLD = 0.2  # Confidence threshold to keep multiple person boxes if more than 1\n\ndef calculate_iou_boxes(box1, box2):\n    \"\"\"Calculate IoU between two boxes [x, y, w, h] in center format\"\"\"\n    b1_x1, b1_x2 = box1['x'] - box1['w']/2, box1['x'] + box1['w']/2\n    b1_y1, b1_y2 = box1['y'] - box1['h']/2, box1['y'] + box1['h']/2\n    b2_x1, b2_x2 = box2['x'] - box2['w']/2, box2['x'] + box2['w']/2\n    b2_y1, b2_y2 = box2['y'] - box2['h']/2, box2['y'] + box2['h']/2\n\n    inter_x1 = max(b1_x1, b2_x1)\n    inter_y1 = max(b1_y1, b2_y1)\n    inter_x2 = min(b1_x2, b2_x2)\n    inter_y2 = min(b1_y2, b2_y2)\n\n    inter_area = max(0, inter_x2 - inter_x1) * max(0, inter_y2 - inter_y1)\n    b1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1)\n    b2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1)\n    union_area = b1_area + b2_area - inter_area\n    return inter_area / union_area if union_area > 0 else 0\n\ndef nms_boxes(boxes, iou_threshold):\n    \"\"\"Apply NMS to boxes of the same class, sorted by confidence\"\"\"\n    if len(boxes) <= 1:\n        return boxes\n    \n    # Sort by confidence (descending)\n    boxes = sorted(boxes, key=lambda x: (x['area']**0.5 * x['conf']), reverse=True)\n    keep = []\n    \n    while boxes:\n        best = boxes.pop(0)\n        keep.append(best)\n        \n        # Remove boxes with high IoU overlap\n        boxes = [b for b in boxes if calculate_iou_boxes(best, b) < iou_threshold]\n    \n    return keep\n\ndef filter_and_nms(pred_str, right_place):\n    \"\"\"Apply area filter, aspect ratio filter, and NMS\"\"\"\n    if pred_str == '-' or pd.isna(pred_str):\n        return pred_str\n    \n    parts = pred_str.strip().split(' ')\n    boxes = []\n    \n    # Parse all boxes\n    for i in range(0, len(parts), 6):\n        if i + 5 < len(parts):\n            cls = int(parts[i])\n            conf = float(parts[i+1])\n            x, y, w, h = float(parts[i+2]), float(parts[i+3]), float(parts[i+4]), float(parts[i+5])\n            area = w * h\n            aspect_ratio = max(w/h, h/w) if min(w, h) > 0 else 999\n            boxes.append({'cls': cls, 'conf': conf, 'x': x, 'y': y, 'w': w, 'h': h, \n                         'area': area, 'aspect_ratio': aspect_ratio})\n    \n    # Separate by class\n    person_boxes = [b for b in boxes if b['cls'] == 0]\n    shemagh_boxes = [b for b in boxes if b['cls'] == 1]\n    other_boxes = [b for b in boxes if b['cls'] not in [0, 1]]\n    has_changed = [len(person_boxes), len(shemagh_boxes)]\n    \n    # Filter persons by area ratio\n    if person_boxes:\n        max_person_area = max(b['area'] for b in person_boxes)\n        person_boxes = [b for b in person_boxes if b['area'] >= MIN_AREA_RATIO * max_person_area]\n\n        if len(person_boxes) > 1:\n            person_boxes = [b for b in person_boxes if (b['conf'] >= MULTIPLE_PERSON_CONF_THRESHOLD and right_place == 0) or (b['conf'] >= 0.3 and right_place == 1)]\n        \n    \n    # Filter persons by aspect ratio\n    person_boxes = [b for b in person_boxes if b['aspect_ratio'] <= MAX_ASPECT_RATIO]\n    shemagh_boxes = [b for b in shemagh_boxes if b['aspect_ratio'] <= MAX_ASPECT_RATIO]\n\n    # Apply NMS to each class separately\n    # person_boxes = nms_boxes(person_boxes, NMS_IOU_THRESHOLD)\n    shemagh_boxes = nms_boxes(shemagh_boxes, NMS_IOU_THRESHOLD)\n    \n    # Combine all boxes\n    filtered_boxes = person_boxes + shemagh_boxes + other_boxes\n    \n    if not filtered_boxes:\n        return '-'\n    \n    # Reconstruct prediction string\n    parts_out = []\n    for b in filtered_boxes:\n        parts_out.append(f\"{b['cls']} {b['conf']:.4f} {b['x']:.4f} {b['y']:.4f} {b['w']:.4f} {b['h']:.4f}\")\n    \n    if has_changed[0] == len(person_boxes) or has_changed[1] == len(shemagh_boxes):\n        return pred_str\n\n    return ' '.join(parts_out)\n\n# Apply filtering to submission\ndf_filtered = df_sub_convnext.copy()\noriginal_predictions = df_filtered['prediction_string'].copy()\ndf_filtered['prediction_string'] = df_filtered.apply(lambda row: filter_and_nms(row['prediction_string'], row['right_place']), axis=1)\n\n# Count changes\nchanged_count = (original_predictions != df_filtered['prediction_string']).sum()\nprint(f\"Modified {changed_count} prediction strings\")\n\n# Detailed stats\ndef count_boxes(pred_str, cls):\n    if pred_str == '-' or pd.isna(pred_str):\n        return 0\n    parts = pred_str.strip().split(' ')\n    return sum(1 for i in range(0, len(parts), 6) if i+5 < len(parts) and int(parts[i]) == cls)\n\norig_persons = sum(count_boxes(p, 0) for p in original_predictions)\nnew_persons = sum(count_boxes(p, 0) for p in df_filtered['prediction_string'])\norig_shemaghs = sum(count_boxes(p, 1) for p in original_predictions)\nnew_shemaghs = sum(count_boxes(p, 1) for p in df_filtered['prediction_string'])\n\nprint(f\"\\nPerson boxes: {orig_persons} -> {new_persons} (removed {orig_persons - new_persons})\")\nprint(f\"Shemagh boxes: {orig_shemaghs} -> {new_shemaghs} (removed {orig_shemaghs - new_shemaghs})\")\n\n# Show examples of changes\nchanged_rows = df_filtered[original_predictions != df_filtered['prediction_string']]\nif len(changed_rows) > 0:\n    print(f\"\\nExamples of filtered images:\")\n    for idx, row in changed_rows.head(180).iterrows():\n        orig = original_predictions.loc[idx]\n        new = row['prediction_string']\n        orig_p = count_boxes(orig, 0)\n        new_p = count_boxes(new, 0)\n        orig_s = count_boxes(orig, 1)\n        new_s = count_boxes(new, 1)\n        # if orig_p!=new_p:\n        if new_s != orig_s or orig_p!=new_p:\n            print(f\"  {row['filename']}: persons {orig_p}->{new_p}, shemaghs {orig_s}->{new_s}\")\n\n# Save filtered submission\ndf_filtered.to_csv('submission.csv', index=False)\nprint(f\"\\nSaved filtered submission to submission_filtered.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}