{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nNow as the GWD Competition has finished and I am happy to announce that our team secured 37th spot out of 2270 teams.\nThanks to my teammates @datahobbit, @ratthachat, @kyoshioka47 and @yashchoudhary for their efforts.\n\nIn this notebook I will demonstrate how we calculated CV for each source for a single model \n\n**NOTE:** This work is inspired by a great [PRIVATE] notebook by @kyoshioka47 for EfficientDet and he is the main architect behind this notebook.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Upvote this notebook or I will delete your Fortnite account.\n\nJust kidding :P \n\nIf you find this notebook useful, please... do whatever you feel like. It's your life.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Setting up the ground","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!cp -rf /kaggle/input/wheatdetection-resnest-develop-branch-july9/wheatdetection/wheatdetection /kaggle/working/\nCODE_PATH = '/kaggle/working/wheatdetection/'\n%cd {CODE_PATH}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport random\nfrom albumentations.pytorch.transforms import ToTensorV2\nsys.path.insert(0, \"/kaggle/input/wheatdetection-resnest-develop-branch-july9/wheatdetection\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\nimport albumentations as A\n\nBEST_PATH = \"/kaggle/input/5fold-68-clear/F1_68_nofinetune_clear_best.bin\"\nVALID_FOLD = 1 #Change your fold here\n\n# print to see best weights information\nckp = torch.load(BEST_PATH)\nprint(ckp.keys())\nprint(ckp['epoch'], ckp['best_valid_loss'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Assign values to hyperparameters\nUSE_NMS = False\nSCORE_THRESHOLD = 0.68\nNMS_IOU_THRESHOLD = 0.5\nIMG_SIZE = 1024\n\nWBF_IOU, WBF_SKIP_BOX = 0.44, 0.38\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom itertools import product\n\nmarking = pd.read_csv('/kaggle/input/global-wheat-detection/train.csv')\n\n\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:, i]\nmarking.drop(columns=['bbox'], inplace=True)\nmarking['area'] = marking['w'] * marking['h']\n\n# Filtering boxes\nmarking = marking[marking['area'] < 154200.0]\nerror_bbox = [100648.0, 145360.0, 149744.0, 119790.0, 106743.0]\nmarking = marking[~marking['area'].isin(error_bbox)]\nmarking = marking[marking['h']>16.0]\nmarking = marking[marking['w']>16.0]\n\n#Stratified 5 fold split\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ndf_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\ndf_folds.loc[:, 'fold'] = 0\n\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n\ntrain_ids = df_folds[df_folds['fold'] != VALID_FOLD].index.values\nvalid_ids = df_folds[df_folds['fold'] == VALID_FOLD].index.values\n\nval_df = marking[marking['image_id'].isin(valid_ids)]\ntrain_df = marking[marking['image_id'].isin(train_ids)]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile ./modeling/wheat_detector.py\n\nimport torch\nfrom torch import nn\nfrom layers import FasterRCNN\nfrom layers.backbone_utils import resnest_fpn_backbone\n\nclass WheatDetector(nn.Module):\n    def __init__(self, cfg, **kwargs):\n        super(WheatDetector, self).__init__()\n        self.backbone = resnest_fpn_backbone(pretrained=False) #change here\n        self.base = FasterRCNN(self.backbone, num_classes=cfg.MODEL.NUM_CLASSES, **kwargs)\n\n    def forward(self, images, targets=None):\n        return self.base(images, targets)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TTA","execution_count":null},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\n\nimport os\nimport warnings\nimport torch\nimport numpy as np\nfrom tqdm import tqdm\nimport pandas as pd\nfrom itertools import product\nimport sys\nsys.path.insert(0, \"./external/wbf\")\nimport ensemble_boxes\nwarnings.filterwarnings(\"ignore\")\nfrom torchvision.transforms import functional_tensor as TF\n\nclass BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n\n    def batch_augment(self, images):\n        raise NotImplementedError\n\n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n\n    def batch_augment(self, images):\n        return images.flip(2)\n\n    def deaugment_boxes(self, boxes):\n        boxes[:, [1, 3]] = self.image_size - boxes[:, [3, 1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(2)\n\n    def batch_augment(self, images):\n        return images.flip(3)\n\n    def deaugment_boxes(self, boxes):\n        boxes[:, [0, 2]] = self.image_size - boxes[:, [2, 0]]\n        return boxes\n\n\nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n\n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0, 2]] = self.image_size - boxes[:, [3, 1]]\n        res_boxes[:, [1, 3]] = boxes[:, [0, 2]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def __init__(self, transforms):\n        self.transforms = transforms\n\n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n\n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n\n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:, 0] = np.min(boxes[:, [0, 2]], axis=1)\n        result_boxes[:, 2] = np.max(boxes[:, [0, 2]], axis=1)\n        result_boxes[:, 1] = np.min(boxes[:, [1, 3]], axis=1)\n        result_boxes[:, 3] = np.max(boxes[:, [1, 3]], axis=1)\n        return result_boxes\n\n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n    \ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None],\n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from evaluate.evaluate import evaluate\n\nclass Tester:\n    def __init__(self, model, device, cfg, test_loader):\n        self.config = cfg\n        self.test_loader = test_loader\n\n        self.base_dir = f'{self.config.OUTPUT_DIR}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.score_threshold = SCORE_THRESHOLD\n        self.iou_threshold = NMS_IOU_THRESHOLD\n        self.use_nms = USE_NMS\n        \n        self.model = model\n        self.model.eval()\n\n        self.device = device\n        self.model.to(self.device)\n\n    def make_tta_predictions(self,images, score_threshold=0.3):\n        self.model.eval()\n        torch.cuda.empty_cache()\n        with torch.no_grad():\n            images = torch.stack(images).float().cuda()\n            predictions = []\n            for tta_transform in tta_transforms:\n                result = []\n                outputs = self.model(tta_transform.batch_augment(images.clone()))\n            for i in range(images.shape[0]):\n                boxes = outputs[i]['boxes'].data.cpu().numpy()  \n                scores = outputs[i]['scores'].data.cpu().numpy()\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes = tta_transform.deaugment_boxes(boxes.copy())\n                result.append({\n                    'boxes': boxes,\n                    'scores': scores[indexes],\n                })\n            predictions.append(result)\n        return predictions\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n\n    def log(self, message):\n        if self.config.VERBOSE:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from config import cfg\n\n#IMPORTANT\ncfg.DATASETS.VALID_FOLD = VALID_FOLD\n\ncfg['OUTPUT_DIR'] = \"/kaggle/working/\"\ncfg['DATASETS']['ROOT_DIR'] = \"/kaggle/input/global-wheat-detection\"\ncfg['TEST']['WEIGHT'] = BEST_PATH\nprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from data import make_data_loader\ntrain_loader, val_loader = make_data_loader(cfg, is_train=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\n\nfrom os import mkdir\nsys.path.append('.')\nfrom config import cfg\nfrom data import make_test_data_loader\nfrom modeling import build_model\nfrom utils.logger import setup_logger\n\n# start here!!\nif True:\n    cfg.freeze()\n\n    output_dir = cfg.OUTPUT_DIR\n    if output_dir and not os.path.exists(output_dir):\n        print('creating ',cfg.OUTPUT_DIR)\n        mkdir(output_dir)\n\n    model = build_model(cfg)\n    device = cfg.MODEL.DEVICE\n    checkpoint = torch.load(cfg.TEST.WEIGHT)\n\n    tester = Tester(model=model, device=device, cfg=cfg, test_loader=val_loader)\n    tester.load(cfg['TEST']['WEIGHT'])\n    print('*** success in loading weights! ***')\n    \n#     predictions = tester.test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utility Functions (Hidden)","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_valid_transforms():\n    return A.Compose(\n        [\n            A.Resize(height=IMG_SIZE, width=IMG_SIZE, p=1.0),\n            \n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\ndef pascal2coco(boxes):\n    boxes = boxes.reshape(-1, 4)\n    boxes[:,2] = (boxes[:,2]-boxes[:,0])\n    boxes[:,3] = (boxes[:,3]-boxes[:,1])\n    return boxes\n\nimport pandas as pd\nimport numpy as np\nimport numba\nimport re\nimport cv2\nimport ast\nimport matplotlib.pyplot as plt\n\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\nfrom collections import namedtuple\nfrom typing import List, Union\n\nBox = namedtuple('Box', 'xmin ymin xmax ymax')\n\n\ndef calculate_iou(gt: List[Union[int, float]],\n                  pred: List[Union[int, float]],\n                  form: str = 'pascal_voc') -> float:\n    \"\"\"Calculates the IoU.\n    \n    Args:\n        gt: List[Union[int, float]] coordinates of the ground-truth box\n        pred: List[Union[int, float]] coordinates of the prdected box\n        form: str gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        IoU: float Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        bgt = Box(gt[0], gt[1], gt[0] + gt[2], gt[1] + gt[3])\n        bpr = Box(pred[0], pred[1], pred[0] + pred[2], pred[1] + pred[3])\n    else:\n        bgt = Box(gt[0], gt[1], gt[2], gt[3])\n        bpr = Box(pred[0], pred[1], pred[2], pred[3])\n        \n\n    overlap_area = 0.0\n    union_area = 0.0\n\n    # Calculate overlap area\n    dx = min(bgt.xmax, bpr.xmax) - max(bgt.xmin, bpr.xmin)\n    dy = min(bgt.ymax, bpr.ymax) - max(bgt.ymin, bpr.ymin)\n\n    if (dx > 0) and (dy > 0):\n        overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (bgt.xmax - bgt.xmin) * (bgt.ymax - bgt.ymin) +\n            (bpr.xmax - bpr.xmin) * (bpr.ymax - bpr.ymin) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\ndef collate_batch(batch):\n    return tuple(zip(*batch))\n\ndef find_best_match(gts, predd, threshold=0.5, form='pascal_voc'):\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n    \n    Args:\n        gts: Coordinates of the available ground-truth boxes\n        pred: Coordinates of the predicted box\n        threshold: Threshold\n        form: Format of the coordinates\n        \n    Return:\n        Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n    \n    for gt_idx, ggt in enumerate(gts):\n        iou = calculate_iou(ggt, predd, form=form)\n        \n        if iou < threshold:\n            continue\n        \n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n\ndef calculate_precision(preds_sorted, gt_boxes, threshold=0.5, form='coco'):\n    \"\"\"Calculates precision per at one threshold.\n    \n    Args:\n        preds_sorted: \n    \"\"\"\n    tp = 0\n    fp = 0\n    fn = 0\n\n    fn_boxes = []\n\n    for pred_idx, pred in enumerate(preds_sorted):\n        best_match_gt_idx = find_best_match(gt_boxes, pred, threshold=threshold, form='coco')\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n\n            # Remove the matched GT box\n            gt_boxes = np.delete(gt_boxes, best_match_gt_idx, axis=0)\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fn += 1\n            fn_boxes.append(pred)\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fp = len(gt_boxes)\n    precision = tp / (tp + fp + fn)\n    return precision, fn_boxes, gt_boxes\n\n\ndef calculate_image_precision(preds_sorted, gt_boxes, thresholds=(0.5), form='coco', debug=False):\n    \n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    for threshold in thresholds:\n        precision_at_threshold, _, _ = calculate_precision(preds_sorted,\n                                                           gt_boxes,\n                                                           threshold=threshold,\n                                                           form=form\n                                                          )\n        if debug:\n            print(\"@{0:.2f} = {1:.4f}\".format(threshold, precision_at_threshold))\n\n        image_precision += precision_at_threshold / n_threshold\n    \n    return image_precision\n\nimport sys\nsys.path.insert(0, \"/kaggle/input/weightedboxesfusion\")\nfrom ensemble_boxes import *\n    \ndef run_wbf(predictions, image_index, image_size=IMG_SIZE, iou_thr=WBF_IOU, skip_box_thr=WBF_SKIP_BOX, weights=None):\n    boxes = [(prediction[image_index]['boxes'] / (image_size - 1)).tolist() for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in\n              predictions]\n    boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels,\n                                                                                    weights=None, iou_thr=iou_thr,\n                                                                                    skip_box_thr=skip_box_thr)\n    boxes = boxes * (image_size - 1)\n    return boxes, scores, labels\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations ","execution_count":null},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nclass train_wheat(Dataset):\n\n    def __init__(self, root, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n        self.root = root\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        p_ratio = random.random()\n        if self.test or p_ratio > 0.7:\n            image, boxes = self.load_image_and_boxes(index)\n        else:\n            if p_ratio < 0.4:\n                image, boxes = self.load_mosaic_image_and_boxes(index)\n            elif p_ratio < 0.55:\n                image, boxes = self.load_image_and_bboxes_with_cutmix(index)\n            else:\n                image, boxes = self.load_mixup_image_and_boxes(index)\n\n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n\n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    # target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{self.root}/train/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return image, boxes\n\n    def load_mosaic_image_and_boxes(self, index, imsize=1024):\n        \"\"\"\n        This implementation of mosaic author:  https://www.kaggle.com/nvnnghia\n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize\n        s = imsize // 2\n\n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y\n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]\n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32)\n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)\n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n\n            boxes[:, 0] += padw\n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n\n        result_boxes = np.concatenate(result_boxes, 0)\n        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[\n            np.where((result_boxes[:, 2] - result_boxes[:, 0]) * (result_boxes[:, 3] - result_boxes[:, 1]) > 0)]\n        return result_image, result_boxes\n\n    def load_image_and_bboxes_with_cutmix(self, index):\n        image, bboxes = self.load_image_and_boxes(index)\n        image_to_be_mixed, bboxes_to_be_mixed = self.load_image_and_boxes(\n            random.randint(0, self.image_ids.shape[0] - 1))\n\n        image_size = image.shape[0]\n        cutoff_x1, cutoff_y1 = [int(random.uniform(image_size * 0.0, image_size * 0.49)) for _ in range(2)]\n        cutoff_x2, cutoff_y2 = [int(random.uniform(image_size * 0.5, image_size * 1.0)) for _ in range(2)]\n\n        image_cutmix = image.copy()\n        image_cutmix[cutoff_y1:cutoff_y2, cutoff_x1:cutoff_x2] = image_to_be_mixed[cutoff_y1:cutoff_y2,\n                                                                 cutoff_x1:cutoff_x2]\n\n        # Begin preparing bboxes_cutmix.\n        # Case 1. Bounding boxes not intersect with cut off patch.\n        bboxes_not_intersect = bboxes[np.concatenate((np.where(bboxes[:, 0] > cutoff_x2),\n                                                      np.where(bboxes[:, 2] < cutoff_x1),\n                                                      np.where(bboxes[:, 1] > cutoff_y2),\n                                                      np.where(bboxes[:, 3] < cutoff_y1)), axis=None)]\n\n        # Case 2. Bounding boxes intersect with cut off patch.\n        bboxes_intersect = bboxes.copy()\n\n        top_intersect = np.where((bboxes[:, 0] < cutoff_x2) &\n                                 (bboxes[:, 2] > cutoff_x1) &\n                                 (bboxes[:, 1] < cutoff_y2) &\n                                 (bboxes[:, 3] > cutoff_y2))\n        right_intersect = np.where((bboxes[:, 0] < cutoff_x2) &\n                                   (bboxes[:, 2] > cutoff_x2) &\n                                   (bboxes[:, 1] < cutoff_y2) &\n                                   (bboxes[:, 3] > cutoff_y1))\n        bottom_intersect = np.where((bboxes[:, 0] < cutoff_x2) &\n                                    (bboxes[:, 2] > cutoff_x1) &\n                                    (bboxes[:, 1] < cutoff_y1) &\n                                    (bboxes[:, 3] > cutoff_y1))\n        left_intersect = np.where((bboxes[:, 0] < cutoff_x1) &\n                                  (bboxes[:, 2] > cutoff_x1) &\n                                  (bboxes[:, 1] < cutoff_y2) &\n                                  (bboxes[:, 3] > cutoff_y1))\n\n        # Remove redundant indices. e.g. a bbox which intersects in both right and top.\n        right_intersect = np.setdiff1d(right_intersect, top_intersect)\n        right_intersect = np.setdiff1d(right_intersect, bottom_intersect)\n        right_intersect = np.setdiff1d(right_intersect, left_intersect)\n        bottom_intersect = np.setdiff1d(bottom_intersect, top_intersect)\n        bottom_intersect = np.setdiff1d(bottom_intersect, left_intersect)\n        left_intersect = np.setdiff1d(left_intersect, top_intersect)\n\n        bboxes_intersect[:, 1][top_intersect] = cutoff_y2\n        bboxes_intersect[:, 0][right_intersect] = cutoff_x2\n        bboxes_intersect[:, 3][bottom_intersect] = cutoff_y1\n        bboxes_intersect[:, 2][left_intersect] = cutoff_x1\n\n        bboxes_intersect[:, 1][top_intersect] = cutoff_y2\n        bboxes_intersect[:, 0][right_intersect] = cutoff_x2\n        bboxes_intersect[:, 3][bottom_intersect] = cutoff_y1\n        bboxes_intersect[:, 2][left_intersect] = cutoff_x1\n\n        bboxes_intersect = bboxes_intersect[np.concatenate((top_intersect,\n                                                            right_intersect,\n                                                            bottom_intersect,\n                                                            left_intersect), axis=None)]\n\n        # Case 3. Bounding boxes inside cut off patch.\n        bboxes_to_be_mixed[:, [0, 2]] = bboxes_to_be_mixed[:, [0, 2]].clip(min=cutoff_x1, max=cutoff_x2)\n        bboxes_to_be_mixed[:, [1, 3]] = bboxes_to_be_mixed[:, [1, 3]].clip(min=cutoff_y1, max=cutoff_y2)\n\n        # Integrate all those three cases.\n        bboxes_cutmix = np.vstack((bboxes_not_intersect, bboxes_intersect, bboxes_to_be_mixed)).astype(int)\n        bboxes_cutmix = bboxes_cutmix[np.where((bboxes_cutmix[:, 2] - bboxes_cutmix[:, 0]) \\\n                                               * (bboxes_cutmix[:, 3] - bboxes_cutmix[:, 1]) > 500)]\n        # End preparing bboxes_cutmix.\n\n        return image_cutmix, bboxes_cutmix\n\n    def load_mixup_image_and_boxes(self, index):\n        image, boxes = self.load_image_and_boxes(index)\n        r_image, r_boxes = self.load_image_and_boxes(random.randint(0, self.image_ids.shape[0] - 1))\n        return (image + r_image) / 2, np.vstack((boxes, r_boxes)).astype(np.int32)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook as tqdm\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\niou_thresholds = [x for x in np.arange(0.5, 0.76, 0.05)]\n\nimport pandas as pd\n\nlog = []\nthresh = 0.5\n\n# get x sources\nimage_source = val_df[['source']].drop_duplicates().to_numpy()\n\n# make df per source\nfor source in image_source:\n    idx = 0\n    precision05 = 0\n    precisions = 0\n    print(\"evaluating for:\", source)\n    source_df = val_df[val_df['source'].isin(source)]\n    source_id = source_df['image_id'].drop_duplicates().to_numpy()\n\n    validation_dataset = train_wheat(\n        root = cfg.DATASETS.ROOT_DIR,\n        image_ids=source_id,\n        marking=source_df,\n        transforms=get_valid_transforms(),\n        test=True\n    )\n    \n    val_loader = DataLoader(\n        validation_dataset,\n        batch_size=1,\n        num_workers=2,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_batch,\n    )\n\n    # load data\n    for j, (images, target, image_ids) in tqdm(enumerate(val_loader)):  \n        # predict\n        # predict\n        tester = Tester(model=model, device=device, cfg=cfg, test_loader=val_loader)\n        predictions = tester.make_tta_predictions(images)\n        ps = []\n        p05 = []\n        for i in range(len(images)):\n            boxes_gt = target[i][\"boxes\"].numpy()\n\n            boxes, scores, labels = run_wbf(predictions, image_index=i, iou_thr=WBF_IOU, skip_box_thr=0.44)\n            boxes = boxes.astype(np.int32).clip(min=0, max=1024)\n\n            # 2coco\n            boxes = pascal2coco(boxes).astype(\"float\")\n            boxes_gt = pascal2coco(boxes_gt)\n\n            preds_sorted_idx = np.argsort(scores)[::-1]\n            boxes_sorted = boxes[preds_sorted_idx]\n\n            precision, fn_boxes, fp_boxes = calculate_precision(boxes_sorted, boxes_gt, threshold=0.5, form='coco')\n            p05.append(precision)\n\n            image_precision = calculate_image_precision(boxes_sorted, boxes_gt,\n                                                thresholds=iou_thresholds,\n                                                form='coco', debug=False)\n            ps.append(image_precision)\n\n        precision05 += np.mean(p05)\n        precisions += np.mean(ps)\n        idx += 1\n\n    print(\"mAP at threshold 0.5: {}\".format(precision05/idx))\n    print(\"mAP at threshold 0.5:0.75: {}\".format(precisions/idx))\n    \n    map50 = precision05/idx\n    map5075 = precisions/idx\n    \n    # evaluate mAP based on each source\n    result = {\"source\": source, \"watermark\": \"FRCNN\", \"map50\": map50, \"map5075\": map5075, \"num_images\": len(source_id)}\n    log.append(result)\n    \ndf = pd.DataFrame(log)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## I will update this notebook soon in the near future \n## Thank you\n","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}