{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Scikit-Opt of WBF params for EfficientDet\n\nHi everyone!\n\nMy name is Alex Shonenkov, I am DL/NLP/CV/TS research engineer. Especially I am in Love with NLP & DL.\n\nRecently I have created kernels for this competition:\n\n- [WBF approach for ensemble](https://www.kaggle.com/shonenkov/wbf-approach-for-ensemble)\n- [[Training] EfficientDet](https://www.kaggle.com/shonenkov/training-efficientdet)\n- [[Inference] EfficientDet](https://www.kaggle.com/shonenkov/inference-efficientdet)\n- [[OOF-Evaluation][Mixup] EfficientDet](https://www.kaggle.com/shonenkov/oof-evaluation-mixup-efficientdet)\n\nThank you all, my friends, for your support, I appreciate it.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Main Idea\n\nRecently I have got boost for this competition using WBF. I used params such as used in paper, but I decided to optimize these params for effdet. It gives me good boost also. \n\n\nI would like to share with you my work about it.\n","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\nimport os\nfrom ensemble_boxes import *\nimport torch\nimport random\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom torch.utils.data import Dataset,DataLoader\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport cv2\nimport gc\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval\nfrom effdet.efficientdet import HeadNet\nfrom sklearn.model_selection import StratifiedKFold\nfrom skopt import gp_minimize, forest_minimize\nfrom skopt.utils import use_named_args\nfrom skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret\nfrom skopt.space import Categorical, Integer, Real\n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"USE_OPTIMIZE = len(glob(f'../input/global-wheat-detection/test/*.jpg')) == 10\nUSE_OPTIMIZE # used for fast inference in submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Folds\n\nI have used holdout fold for optimizing. Here you can see used splitting.","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"marking = pd.read_csv('../input/global-wheat-detection/train.csv')\n\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:,i]\nmarking.drop(columns=['bbox'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%%time\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ndf_folds = marking[['image_id']]\ndf_folds['bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()\ndf_folds['source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds['stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\n\ntrain_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))\ndf_holdout = df_folds.iloc[holdout_index].copy()\ndf_folds = df_folds.iloc[train_index].copy()\n\ndf_folds['fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Holdout Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_valid_transforms():\n    return A.Compose(\n        [\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n\nTRAIN_ROOT_PATH = '../input/global-wheat-detection/train'\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image, boxes = self.load_image_and_boxes(index)\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return image, boxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"holdout_dataset = DatasetRetriever(\n    image_ids=df_holdout.index.values,\n    marking=marking,\n    transforms=get_valid_transforms(),\n    test=True,\n)\n\nholdout_loader = DataLoader(\n    holdout_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=2,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Models\n\nFor prediction of holdout fold I have trained the another remaining 5folds  ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_net(checkpoint_path):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint)\n\n    del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\nif USE_OPTIMIZE:\n    models = [\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n    ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Holdout Prediction\n\nLets make prediction of holdout fold, just like hidden test in this competition. Models didn't use holdout data for training!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_det(index, det, score_threshold=0.25):\n    boxes = det[index].detach().cpu().numpy()[:,:4]    \n    scores = det[index].detach().cpu().numpy()[:,4]\n    boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n    boxes = (boxes*2).clip(min=0, max=1023).astype(int)\n    indexes = np.where(scores>score_threshold)\n    boxes = boxes[indexes]\n    scores = scores[indexes]\n    return boxes, scores\n\n\nif USE_OPTIMIZE:\n    all_predictions = []\n\n    for images, targets, image_ids in tqdm(holdout_loader, total=len(holdout_loader)):\n        with torch.no_grad():\n            images = torch.stack(images)\n            images = images.cuda().float()\n\n            fold_predictions = {}\n            for fold_number in range(5):\n                fold_predictions[fold_number] = models[fold_number](images, torch.tensor([1]*images.shape[0]).float().cuda())\n\n            for i in range(images.shape[0]):\n                image_predictions = {\n                    'image_id': image_ids[i],\n                    'gt_boxes': (targets[i]['boxes'].cpu().numpy()*2).clip(min=0, max=1023).astype(int),\n                }\n                for fold_number in range(5):\n                    boxes, scores = process_det(i, fold_predictions[fold_number])\n                    image_predictions[f'pred_boxes_fold{fold_number}'] = boxes\n                    image_predictions[f'scores_fold{fold_number}'] = scores\n\n                all_predictions.append(image_predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation\n\nHere I have used [really good evaluation scripts](https://www.kaggle.com/pestipeti/competition-metric-details-script) by [Peter](https://www.kaggle.com/pestipeti), I recommend to use it! ","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport numba\nimport re\nimport cv2\nimport ast\nimport matplotlib.pyplot as plt\n\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision\n\ndef show_result(sample_id, preds, gt_boxes):\n    sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)\n    sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for pred_box in preds:\n        cv2.rectangle(\n            sample,\n            (pred_box[0], pred_box[1]),\n            (pred_box[2], pred_box[3]),\n            (220, 0, 0), 2\n        )\n\n    for gt_box in gt_boxes:    \n        cv2.rectangle(\n            sample,\n            (gt_box[0], gt_box[1]),\n            (gt_box[2], gt_box[3]),\n            (0, 0, 220), 2\n        )\n\n    ax.set_axis_off()\n    ax.imshow(sample)\n    ax.set_title(\"RED: Predicted | BLUE - Ground-truth\")\n    \n# Numba typed list!\niou_thresholds = numba.typed.List()\n\nfor x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n    iou_thresholds.append(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_final_score(\n    all_predictions,\n    iou_thr,\n    skip_box_thr,\n    method, # weighted_boxes_fusion, nms, soft_nms, non_maximum_weighted\n    sigma=0.5,\n):\n    final_scores = []\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        image_id = all_predictions[i]['image_id']\n        folds_boxes, folds_scores, folds_labels = [], [], []\n        for fold_number in range(5):\n            pred_boxes = all_predictions[i][f'pred_boxes_fold{fold_number}'].copy()\n            scores = all_predictions[i][f'scores_fold{fold_number}'].copy()\n            folds_boxes.append(pred_boxes)\n            folds_scores.append(scores)\n            folds_labels.append(np.ones(pred_boxes.shape[0]))\n        \n        if method == 'weighted_boxes_fusion':\n            boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n        elif method == 'nms':\n            boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)\n        elif method == 'soft_nms':\n            boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)\n        elif method == 'non_maximum_weighted':\n            boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n        else:\n            raise\n        image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')\n        final_scores.append(image_precision)\n\n    return np.mean(final_scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# standart params as here https://github.com/ZFTurbo/Weighted-Boxes-Fusion\nif USE_OPTIMIZE:\n    print('[WBF]: ', calculate_final_score(\n        all_predictions, \n        iou_thr=0.55,\n        skip_box_thr=0.0001,\n        method='weighted_boxes_fusion',\n    ))\n    print('[NMS]: ', calculate_final_score(\n        all_predictions, \n        iou_thr=0.55,\n        skip_box_thr=0.0001,\n        method='nms',\n    ))\n    print('[SOFT NMS]: ', calculate_final_score(\n        all_predictions, \n        iou_thr=0.55,\n        skip_box_thr=0.0001,\n        sigma=0.1,\n        method='soft_nms',\n    ))\n    print('[NMW]: ', calculate_final_score(\n        all_predictions, \n        iou_thr=0.55,\n        skip_box_thr=0.0001,\n        method='non_maximum_weighted',\n    ))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# [Bayesian optimization using Gaussian Processes](https://scikit-optimize.github.io/stable/modules/generated/skopt.gp_minimize.html)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def log(text):\n    with open('opt.log', 'a+') as logger:\n        logger.write(f'{text}\\n')\n\ndef optimize(space, all_predictions, method, n_calls=10):\n    @use_named_args(space)\n    def score(**params):\n        log('-'*5 + f'{method}' + '-'*5)\n        log(params)\n        final_score = calculate_final_score(all_predictions, method=method, **params)\n        log(f'final_score = {final_score}')\n        log('-'*10)\n        return -final_score\n\n    return gp_minimize(func=score, dimensions=space, n_calls=n_calls)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NMS","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nspace = [\n    Real(0, 1, name='iou_thr'),\n    Real(0.0, 1, name='skip_box_thr'),\n]\n\nif USE_OPTIMIZE:\n    opt_result = optimize(\n        space, \n        all_predictions,\n        method='nms',\n        n_calls=50,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_convergence(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_regret(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_objective(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_evaluations(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    best_final_score = -opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\n    print('-'*13 + 'NMS' + '-'*14)\n    print(f'[Best Iou Thr]: {best_iou_thr:.3f}')\n    print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\n    print(f'[Best Score]: {best_final_score:.4f}')\n    print('-'*30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Soft NMS","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nspace = [\n    Real(0, 1, name='iou_thr'),\n    Real(0.0, 1, name='skip_box_thr'),\n    Real(0.0, 1, name='sigma'),\n]\n\nif USE_OPTIMIZE:\n    opt_result = optimize(\n        space, \n        all_predictions,\n        method='soft_nms',\n        n_calls=50,\n    )","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_convergence(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_regret(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_objective(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_evaluations(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    best_final_score = -opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\n    best_sigma = opt_result.x[2]\n    print('-'*11 + 'SOFT NMS' + '-'*11)\n    print(f'[Best Iou Thr]: {best_iou_thr:.3f}')\n    print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\n    print(f'[Best Sigma]: {best_sigma:.3f}')\n    print(f'[Best Score]: {best_final_score:.4f}')\n    print('-'*30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NMW","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nspace = [\n    Real(0, 1, name='iou_thr'),\n    Real(0.0, 1, name='skip_box_thr'),\n]\n\nif USE_OPTIMIZE:\n    opt_result = optimize(\n        space, \n        all_predictions,\n        method='non_maximum_weighted',\n        n_calls=50,\n    )","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_convergence(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_regret(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_objective(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_evaluations(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    best_final_score = -opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\n    print('-'*13 + 'NMW' + '-'*14)\n    print(f'[Best Iou Thr]: {best_iou_thr:.3f}')\n    print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\n    print(f'[Best Score]: {best_final_score:.4f}')\n    print('-'*30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# WBF","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nspace = [\n    Real(0, 1, name='iou_thr'),\n    Real(0.25, 1, name='skip_box_thr'),\n]\n\nif USE_OPTIMIZE:\n    opt_result = optimize(\n        space, \n        all_predictions,\n        method='weighted_boxes_fusion',\n        n_calls=50,\n    )","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_convergence(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_regret(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_objective(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    plot_evaluations(opt_result);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    best_final_score = -opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\nelse:\n    # calculated early for fast inference in submission [see version 1 with n_calls=300]\n    best_final_score = 0.6953\n    best_iou_thr = 0.44\n    best_skip_box_thr = 0.43\n\nprint('-'*13 + 'WBF' + '-'*14)\nprint(f'[Best Iou Thr]: {best_iou_thr:.3f}')\nprint(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\nprint(f'[Best Score]: {best_final_score:.4f}')\nprint('-'*30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference with best barams","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n        load_net('../input/wheat-effdet5-fold0-best-checkpoint/fold0-best.bin'),\n    ]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"DATA_ROOT_PATH = '../input/global-wheat-detection/test'\n\nclass TestDatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\ndef get_valid_transforms():\n    return A.Compose(\n        [\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n    )\n\ndataset = TestDatasetRetriever(\n    image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),\n    transforms=get_valid_transforms()\n)\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndata_loader = DataLoader(\n    dataset,\n    batch_size=2,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)\n\ndef make_predictions(\n    images, \n    score_threshold=0.25,\n):\n    images = torch.stack(images).cuda().float()\n    predictions = []\n    for fold_number, net in enumerate(models):\n        with torch.no_grad():\n            det = net(images, torch.tensor([1]*images.shape[0]).float().cuda())\n            result = []\n            for i in range(images.shape[0]):\n                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                scores = det[i].detach().cpu().numpy()[:,4]\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                result.append({\n                    'boxes': boxes[indexes],\n                    'scores': scores[indexes],\n                })\n            predictions.append(result)\n    return predictions\n\n\ndef run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)\n\n\nresults = []\n\nfor images, image_ids in data_loader:\n    predictions = make_predictions(images)\n    for i, image in enumerate(images):\n        boxes, scores, labels = run_wbf(predictions, image_index=i)\n        boxes = (boxes*2).astype(np.int32).clip(min=0, max=1023)\n        image_id = image_ids[i]\n        \n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n        results.append(result)\n        \n        \ntest_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you for reading my kernel!","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}