{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":19989,"databundleVersionId":1160143,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ultralytics --no-deps --upgrade\n!pip install \"decord\"\n!pip install \"ftfy==6.1.1\"\n!pip install \"iopath>=0.1.10\"\n!git clone -b apple-silicon-support https://github.com/provos/sam3.git # have fixes for CPU runtime\n!mv sam3 sam3_repo","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nimport huggingface_hub as hf\nimport wandb\n\n#### login to huggingface\nuser_secrets = UserSecretsClient()\nhf_token = user_secrets.get_secret(\"HF_TOKEN\")\nhf.login(token=hf_token)\n\n#### login to wandb\n!yolo settings wandb=True\nwandb_key = user_secrets.get_secret(\"wandbKey\")\nwandb.login(key=wandb_key)\n\n#### add repository in the system path\nimport sys\nsys.path.insert(0, \"./sam3_repo\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preperation","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport os\n\nRANDOM_SEED = 9235789\nANNOTATION_PATH = \"/kaggle/input/global-wheat-detection/train.csv\"\nDATASET_BASE = Path(\"/kaggle/working/global_wheat_detection\")\nTRAIN_IMAGE_DIR = DATASET_BASE / \"images\" / \"train\"\nTRAIN_LABEL_DIR = DATASET_BASE / \"labels\" / \"train\"\nVAL_IMAGE_DIR = DATASET_BASE / \"images\" / \"val\"\nVAL_LABEL_DIR = DATASET_BASE / \"labels\" / \"val\"\nSRC_IMAGE_DIR = Path(\"/kaggle/input/global-wheat-detection/train\")\n\nDATA_CONFIG = \"./dataset_config.yml\"\nPROJECT_NAME = \"SAM3 Comparisons\"\nEXPERIMENT_NAME = \"global_wheat_detection_yolo_v11\"\n\n# create the directoris\nos.makedirs(TRAIN_IMAGE_DIR, exist_ok=True)\nos.makedirs(TRAIN_LABEL_DIR, exist_ok=True)\nos.makedirs(VAL_IMAGE_DIR, exist_ok=True)\nos.makedirs(VAL_LABEL_DIR, exist_ok=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import List, Callable, Union\nfrom tqdm.notebook import tqdm\nimport pandas as pd\nimport shutil\nimport os\n\ndef convert_to_yolo_txts(df: pd.DataFrame, out_dir: Path, class_id: int = 0):\n    for _, row in df.iterrows():\n        image_id = row[\"image_id\"]\n        img_w, img_h = row[\"width\"], row[\"height\"]\n        xmin, ymin, bw, bh = eval(row[\"bbox\"])\n\n        # convert to YOLO normalized format\n        x_center = (xmin + bw / 2) / img_w\n        y_center = (ymin + bh / 2) / img_h\n        w_norm = bw / img_w\n        h_norm = bh / img_h\n\n        # write the line to respective files\n        line = f\"{class_id} {x_center} {y_center} {w_norm} {h_norm}\"\n        txt_path = out_dir / f\"{image_id}.txt\"\n        with open(txt_path, \"a\") as f:\n            f.write(line + \"\\n\")\n\ndef copy_images(image_ids: List[str], src_path: Path, dest_path: Path):\n    with ThreadPoolExecutor() as executor:\n        futures = []\n        for image_id in image_ids:\n            src_image = src_path / f\"{image_id}.jpg\"\n            dst_image = dest_path / f\"{image_id}.jpg\"\n            if not dst_image.exists():\n                future = executor.submit(shutil.copy, src_image, dst_image)\n                futures.append(future)\n\n        with tqdm(total=len(futures)) as pbar:\n            for future in as_completed(futures):\n                try:\n                    future.result()\n                    pbar.update()\n                except Exception as err:\n                    executor.shutdown(wait=False, cancel_futures=True)\n                    raise err\n\n    return True","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport yaml\n\n\n# load full annotations\nannotation_df = pd.read_csv(ANNOTATION_PATH)\n\n# split the data frame into train or test columns\ntrain_image_ids, val_image_ids = train_test_split(annotation_df['image_id'].unique(), test_size=0.1, random_state=RANDOM_SEED)\ntrain_df = annotation_df[annotation_df['image_id'].isin(train_image_ids)]\nval_df = annotation_df[annotation_df['image_id'].isin(val_image_ids)]\n\n# copy the images to the respective directories\ncopy_images(train_image_ids, src_path=SRC_IMAGE_DIR, dest_path=TRAIN_IMAGE_DIR)\ncopy_images(val_image_ids, src_path=SRC_IMAGE_DIR, dest_path=VAL_IMAGE_DIR)\n\n# convert annotations to txts\nconvert_to_yolo_txts(train_df, out_dir=TRAIN_LABEL_DIR)\nconvert_to_yolo_txts(val_df, out_dir=VAL_LABEL_DIR)\n\n# save data config file\ndata_config = {\n    \"path\": str(DATASET_BASE),\n    \"train\": \"images/train\",\n    \"val\": \"images/val\",\n    \"names\": {0: \"wheat_head\"}\n}\n\nwith open(DATA_CONFIG, \"w\") as f:\n    yaml.dump(data_config, f, sort_keys=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\n\nmodel = YOLO(\"yolo11l.pt\")\nresults = model.train(\n    data=DATA_CONFIG, \n    project=PROJECT_NAME, \n    name=EXPERIMENT_NAME,\n    seed=RANDOM_SEED,\n    half=True, \n    exist_ok=True,\n    amp=True,\n    imgsz=768,\n    freeze=None,\n    single_cls=True,\n    epochs=30,\n    patience=10,\n    close_mosaic=5,\n    batch=12,\n    optimizer='Adam',\n    lr0=0.001,\n    degrees=20,\n    translate=0.2,\n    flipud=0.5,\n    fliplr=0.5\n)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-29T18:24:29.756Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation of the trained model","metadata":{}},{"cell_type":"code","source":"from pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval\nfrom sam3.model.sam3_image_processor import Sam3Processor\nfrom sam3.model_builder import build_sam3_image_model\nfrom functools import partial\nfrom PIL import Image\nimport json\nimport torch\n\n\ndef evaluate_coco(coco_gt, coco_pred):\n    coco_gt = COCO(coco_gt)\n    coco_dt = coco_gt.loadRes(coco_pred)\n    coco_eval = COCOeval(coco_gt, coco_dt, \"bbox\")\n    coco_eval.evaluate()\n    coco_eval.accumulate()\n    coco_eval.summarize()\n\n    metrics = {\n        \"AP\": coco_eval.stats[0],\n        \"AP50\": coco_eval.stats[1],\n        \"AP75\": coco_eval.stats[2],\n        \"AP_small\": coco_eval.stats[3],\n        \"AP_medium\": coco_eval.stats[4],\n        \"AP_large\": coco_eval.stats[5],\n        \"AR_1\": coco_eval.stats[6],\n        \"AR_10\": coco_eval.stats[7],\n        \"AR_100\": coco_eval.stats[8],\n        \"AR_small\": coco_eval.stats[9],\n        \"AR_medium\": coco_eval.stats[10],\n        \"AR_large\": coco_eval.stats[11],\n    }\n    return metrics\n\n\ndef generate_coco_formatted_datasets(infer_func: Callable, df: pd.DataFrame, image_dir: Path):\n    coco_gt = {\n        \"info\": {},\n        \"images\": [],\n        \"annotations\": [],\n        \"categories\": [{\"id\": 0, \"name\": \"wheat_head\"}]\n    }\n    \n    coco_preds = []\n    image_id, ann_id = 1, 1\n    grouped = df.groupby(\"image_id\")\n    for image_file_id, rows in tqdm(grouped):\n        coco_gt[\"images\"].append({\n            \"id\": image_id,\n            \"file_name\": f\"{image_file_id}.jpg\",\n            \"width\": int(rows[\"width\"].iloc[0]),\n            \"height\": int(rows[\"height\"].iloc[0])\n        })\n\n        # add annotations\n        for _, row in rows.iterrows():\n            x1, y1, w, h = eval(row[\"bbox\"])\n            coco_gt[\"annotations\"].append({\n                \"id\": ann_id,\n                \"image_id\": image_id,\n                \"category_id\": 0,\n                \"bbox\": [x1, y1, w, h],\n                \"area\": w * h,\n                \"iscrowd\": 0\n            })\n            ann_id += 1\n\n        # inference and accumulate predictions\n        image_path = image_dir / f\"{image_file_id}.jpg\"\n        coco_preds += infer_func(image_path=image_path, image_id=image_id)\n        image_id += 1\n\n    return coco_gt, coco_preds\n\ndef nms(boxes, scores, iou_threshold=0.5):\n    # boxes: (N, 4) in xyxy\n    # scores: (N,)\n    idxs = scores.argsort(descending=True)\n    keep = []\n\n    while idxs.numel() > 0:\n        current = idxs[0].item()\n        keep.append(current)\n\n        if idxs.numel() == 1:\n            break\n\n        rest = idxs[1:]\n\n        x1 = torch.maximum(boxes[current, 0], boxes[rest, 0])\n        y1 = torch.maximum(boxes[current, 1], boxes[rest, 1])\n        x2 = torch.minimum(boxes[current, 2], boxes[rest, 2])\n        y2 = torch.minimum(boxes[current, 3], boxes[rest, 3])\n\n        inter_w = torch.clamp(x2 - x1, min=0)\n        inter_h = torch.clamp(y2 - y1, min=0)\n        inter = inter_w * inter_h\n\n        area_current = (boxes[current, 2] - boxes[current, 0]) * \\\n                       (boxes[current, 3] - boxes[current, 1])\n        area_rest = (boxes[rest, 2] - boxes[rest, 0]) * \\\n                    (boxes[rest, 3] - boxes[rest, 1])\n\n        union = area_current + area_rest - inter\n        iou = inter / union\n\n        idxs = rest[iou < iou_threshold]\n\n    return torch.tensor(keep, dtype=torch.long)\n\ndef perform_non_max_suppression(outputs, iou_threshold: float = 0.5):\n    keep_idx = nms(outputs['boxes'], outputs['scores'], iou_threshold=iou_threshold)\n    outputs['scores'] = outputs['scores'][keep_idx]\n    outputs['boxes'] = outputs['boxes'][keep_idx, :]\n    outputs['masks'] = outputs['masks'][keep_idx, :, :]\n    return outputs\n\ndef run_sam3(processor: Sam3Processor, image: Union[str, Image], prompt: str):\n    if isinstance(image, (str, Path)):\n        image = Image.open(image)\n\n    # run the model\n    inference_state  = processor.set_image(image)\n    output = processor.set_text_prompt(state=inference_state, prompt=prompt)\n    return output","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"########################### Benchmark the best trained model\ndef infer_yolo_model(model, image_path: Path, image_id: int):\n    image_preds = []\n    results = model.predict(image_path, verbose=False)[0]\n    for pred in results.boxes:\n        xmin, ymin, xmax, ymax = pred.xyxy[0].tolist()\n        image_preds.append({\n            \"image_id\": image_id,\n            \"category_id\": int(pred.cls),\n            \"bbox\": [xmin, ymin, xmax - xmin, ymax - ymin],\n            \"score\": float(pred.conf)\n        })\n\n    return image_preds\n\n# load the model\nbest_model_weight = os.path.join(PROJECT_NAME, EXPERIMENT_NAME, 'weights', 'best.pt')\nbest_model = YOLO(best_model_weight)\n\n# do the inference on validation dataset to get benchmarks\ninfer_func = partial(infer_yolo_model, model=best_model)\ncoco_gt, coco_yolo_dt = generate_coco_formatted_datasets(infer_func, val_df, VAL_IMAGE_DIR)\n\n# save the ground truth and detection files\ncoco_yolo_gt_path = os.path.join(PROJECT_NAME, EXPERIMENT_NAME,  \"global_wheat_yolo_gt.json\")\ncoco_yolo_dt_path = os.path.join(PROJECT_NAME, EXPERIMENT_NAME,  \"global_wheat_yolo_dt.json\")\nwith open(coco_yolo_gt_path, 'w') as file:\n    json.dump(coco_gt, file)\nwith open(coco_yolo_dt_path, 'w') as file:\n    json.dump(coco_yolo_dt, file)\n\n# evaluate and record the metrics\nyolo_metrics = evaluate_coco(coco_yolo_gt_path, coco_yolo_dt_path)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"########################### Benchmark the SAM3 model\n\ndef infer_sam3_model(model, image_path: Path, image_id: int):\n    image_preds = []\n    sam3_predictions = run_sam3(model, image_path, prompt=\"wheat, flower, seed head, green seed head, spike, green spike\")\n    sam3_predictions = perform_non_max_suppression(sam3_predictions, iou_threshold=0.5)\n    for score, bbox in zip(sam3_predictions['scores'].tolist(), sam3_predictions['boxes'].tolist()):\n        xmin, ymin, xmax, ymax = bbox\n        image_preds.append({\n            \"image_id\": image_id,\n            \"category_id\": 0,\n            \"bbox\": [int(xmin), int(ymin), int(xmax - xmin), int(ymax - ymin)],\n            \"score\": float(score)\n        })\n\n    return image_preds\n\n\n### build SAM3 model\nmodel = build_sam3_image_model(enable_segmentation=True)\nprocessor = Sam3Processor(model, confidence_threshold=0.3)\n\n# do the inference on validation dataset to get benchmarks\ninfer_func = partial(infer_sam3_model, model=processor)\ncoco_gt, coco_sam_dt = generate_coco_formatted_datasets(infer_func, val_df, VAL_IMAGE_DIR)\n\n# save the ground truth and detection files\ncoco_sam3_gt_path = \"./global_wheat_sam3_gt.json\"\ncoco_sam3_dt_path = \"./global_wheat_sam3_dt.json\"\nwith open(coco_sam3_gt_path, 'w') as file:\n    json.dump(coco_gt, file)\nwith open(coco_sam3_dt_path, 'w') as file:\n    json.dump(coco_sam_dt, file)\n\n# evaluate and record the metrics\nsam3_metrics = evaluate_coco(coco_sam3_gt_path, coco_sam3_dt_path)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Comparision between yolo and SAM3","metadata":{}},{"cell_type":"code","source":"from IPython.display import display, HTML\nimport pandas as pd\n\ndef compare_coco_metrics(metrics_a, metrics_b, name_a=\"Model A\", name_b=\"Model B\"):\n    def safe(v):\n        return 0 if v == -1 else v\n\n    df = pd.DataFrame([\n        {\n            \"Metric\": key,\n            name_a: round(safe(metrics_a[key]), 4),\n            name_b: round(safe(metrics_b[key]), 4),\n            \"Δ (B - A)\": round(safe(metrics_b[key]) - safe(metrics_a[key]), 4),\n            \"% Change\": (\n                round(((safe(metrics_b[key]) - safe(metrics_a[key])) / safe(metrics_a[key])) * 100, 2)\n                if safe(metrics_a[key]) != 0 else None\n            )\n        }\n        for key in metrics_a.keys()\n    ])\n\n    # Pretty colors\n    def color_diff(val):\n        if val is None:\n            return \"\"\n        if val > 0:\n            return \"background-color: #b2fab4\"\n        elif val < 0:\n            return \"background-color: #ffb3b3\"\n        return \"\"\n\n    styled = (\n        df.style\n        .applymap(color_diff, subset=[\"Δ (B - A)\", \"% Change\"])\n        .set_properties(**{\"text-align\": \"center\"})\n        .hide(axis=\"index\")\n    )\n\n    display(styled)\n\n    # Scoring weights\n    weights = {\n        \"AP\": 3.0, \"AP50\": 2.0, \"AP75\": 2.0,\n        \"AP_small\": 0.7, \"AP_medium\": 0.5, \"AP_large\": 0.2,\n        \"AR_100\": 3.0, \"AR_1\": 0.1, \"AR_10\": 0.1,\n        \"AR_small\": 0.7, \"AR_medium\": 0.5, \"AR_large\": 0.2\n    }\n\n    score_a, score_b = 0, 0\n    for key in metrics_a.keys():\n        w = weights.get(key, 1.0)\n        score_a += safe(metrics_a[key]) * w\n        score_b += safe(metrics_b[key]) * w\n\n    diff_total = score_b - score_a\n    pct_change_total = (diff_total / score_a * 100) if score_a != 0 else None\n\n    print(\"------------------------------------------------------------\")\n    print(\"🏆 **Overall Comparison**\")\n    print(f\"{name_a} Score: {score_a:.4f}\")\n    print(f\"{name_b} Score: {score_b:.4f}\")\n    print(f\"Difference (B - A): {diff_total:.4f}\")\n\n    if pct_change_total is not None:\n        print(f\"Percentage Difference: {pct_change_total:.2f}%\")\n    else:\n        print(\"Percentage Difference: N/A\")\n\n    if diff_total > 0:\n        print(f\"\\n🔥 **{name_b} is overall better by {diff_total:.4f} points ({pct_change_total:.2f}%).**\")\n    elif diff_total < 0:\n        print(f\"\\n🔥 **{name_a} is overall better by {-diff_total:.4f} points ({-pct_change_total:.2f}%).**\")\n    else:\n        print(\"\\n🤝 Both models are equally good (tie).\")\n    print(\"------------------------------------------------------------\")\n\n    return df\n\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparision = compare_coco_metrics(yolo_metrics, sam3_metrics, name_a='yolov11-nano', name_b='SAM3')\ncomparision.to_csv('results.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}