{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-06-20T15:03:50.966850Z","iopub.execute_input":"2021-06-20T15:03:50.967161Z","iopub.status.idle":"2021-06-20T15:04:36.386418Z","shell.execute_reply.started":"2021-06-20T15:03:50.967132Z","shell.execute_reply":"2021-06-20T15:04:36.385408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\nimport time\nfrom tqdm.notebook import tqdm\nfrom ast import literal_eval\nimport torch\nimport numba\nfrom numba import jit\nfrom typing import List, Union, Tuple\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom torch.utils.data import Dataset,DataLoader\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport cv2\nimport gc\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval\nfrom effdet.efficientdet import HeadNet","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:11:19.224566Z","iopub.execute_input":"2021-06-20T15:11:19.224914Z","iopub.status.idle":"2021-06-20T15:11:19.401909Z","shell.execute_reply.started":"2021-06-20T15:11:19.224861Z","shell.execute_reply":"2021-06-20T15:11:19.401159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:04:40.596908Z","iopub.execute_input":"2021-06-20T15:04:40.597233Z","iopub.status.idle":"2021-06-20T15:04:40.625411Z","shell.execute_reply.started":"2021-06-20T15:04:40.597198Z","shell.execute_reply":"2021-06-20T15:04:40.623522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"box1 = np.array([1.0, 1.0, 10.0, 10.0])\nbox2 = np.array([2.0, 1.0, 10.0, 10.0])\n\ncalculate_iou(box1, box2, form='pascal_voc')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:04:40.626663Z","iopub.execute_input":"2021-06-20T15:04:40.627075Z","iopub.status.idle":"2021-06-20T15:04:41.494151Z","shell.execute_reply.started":"2021-06-20T15:04:40.627037Z","shell.execute_reply":"2021-06-20T15:04:41.492646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef bb_intersection_over_union(A, B):\n    xA = max(A[0], B[0])\n    yA = max(A[1], B[1])\n    xB = min(A[2], B[2])\n    yB = min(A[3], B[3])\n\n    # compute the area of intersection rectangle\n    interArea = max(0, xB - xA) * max(0, yB - yA)\n\n    if interArea == 0:\n        return 0.0\n\n    # compute the area of both the prediction and ground-truth rectangles\n    boxAArea = (A[2] - A[0]) * (A[3] - A[1])\n    boxBArea = (B[2] - B[0]) * (B[3] - B[1])\n\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n    return iou\n\n\ndef prefilter_boxes(boxes, scores, labels, weights, thr):\n    # Create dict with boxes stored by its label\n    new_boxes = dict()\n    for t in range(len(boxes)):\n        for j in range(len(boxes[t])):\n            label = int(labels[t][j])\n            score = scores[t][j]\n            if score < thr:\n                break\n            box_part = boxes[t][j]\n            b = [int(label), float(score) * weights[t], float(box_part[0]), float(box_part[1]), float(box_part[2]), float(box_part[3])]\n            if label not in new_boxes:\n                new_boxes[label] = []\n            new_boxes[label].append(b)\n\n    # Sort each list in dict and transform it to numpy array\n    for k in new_boxes:\n        current_boxes = np.array(new_boxes[k])\n        new_boxes[k] = current_boxes[current_boxes[:, 1].argsort()[::-1]]\n\n    return new_boxes\n\n\ndef get_weighted_box(boxes, conf_type='avg'):\n    \"\"\"\n    Create weighted box for set of boxes\n    :param boxes: set of boxes to fuse \n    :param conf_type: type of confidence one of 'avg' or 'max'\n    :return: weighted box\n    \"\"\"\n\n    box = np.zeros(6, dtype=np.float32)\n    conf = 0\n    conf_list = []\n    for b in boxes:\n        box[2:] += (b[1] * b[2:])\n        conf += b[1]\n        conf_list.append(b[1])\n    box[0] = boxes[0][0]\n    if conf_type == 'avg':\n        box[1] = conf / len(boxes)\n    elif conf_type == 'max':\n        box[1] = np.array(conf_list).max()\n    box[2:] /= conf\n    return box\n\n\ndef find_matching_box(boxes_list, new_box, match_iou):\n    best_iou = match_iou\n    best_index = -1\n    for i in range(len(boxes_list)):\n        box = boxes_list[i]\n        if box[0] != new_box[0]:\n            continue\n        iou = bb_intersection_over_union(box[2:], new_box[2:])\n        if iou > best_iou:\n            best_index = i\n            best_iou = iou\n\n    return best_index, best_iou\n\n\ndef weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=None, iou_thr=0.55, skip_box_thr=0.0, conf_type='avg', allows_overflow=False):\n    '''\n    :param boxes_list: list of boxes predictions from each model, each box is 4 numbers. \n    It has 3 dimensions (models_number, model_preds, 4)\n    Order of boxes: x1, y1, x2, y2. We expect float normalized coordinates [0; 1]\n    :param scores_list: list of scores for each model \n    :param labels_list: list of labels for each model\n    :param weights: list of weights for each model. Default: None, which means weight == 1 for each model\n    :param intersection_thr: IoU value for boxes to be a match\n    :param skip_box_thr: exclude boxes with score lower than this variable  \n    :param conf_type: how to calculate confidence in weighted boxes. 'avg': average value, 'max': maximum value\n    :param allows_overflow: false if we want confidence score not exceed 1.0 \n    \n    :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2). \n    :return: scores: confidence scores\n    :return: labels: boxes labels\n    '''\n\n    if weights is None:\n        weights = np.ones(len(boxes_list))\n    if len(weights) != len(boxes_list):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights), len(boxes_list)))\n        weights = np.ones(len(boxes_list))\n    weights = np.array(weights)\n\n    if conf_type not in ['avg', 'max']:\n        print('Unknown conf_type: {}. Must be \"avg\" or \"max\"'.format(conf_type))\n        exit()\n\n    filtered_boxes = prefilter_boxes(boxes_list, scores_list, labels_list, weights, skip_box_thr)\n    if len(filtered_boxes) == 0:\n        return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,))\n\n    overall_boxes = []\n    for label in filtered_boxes:\n        boxes = filtered_boxes[label]\n        new_boxes = []\n        weighted_boxes = []\n\n        # Clusterize boxes\n        for j in range(0, len(boxes)):\n            index, best_iou = find_matching_box(weighted_boxes, boxes[j], iou_thr)\n            if index != -1:\n                new_boxes[index].append(boxes[j])\n                weighted_boxes[index] = get_weighted_box(new_boxes[index], conf_type)\n            else:\n                new_boxes.append([boxes[j].copy()])\n                weighted_boxes.append(boxes[j].copy())\n\n        # Rescale confidence based on number of models and boxes\n        for i in range(len(new_boxes)):\n            if not allows_overflow:\n                weighted_boxes[i][1] = weighted_boxes[i][1] * min(weights.sum(), len(new_boxes[i])) / weights.sum()\n            else:\n                weighted_boxes[i][1] = weighted_boxes[i][1] * len(new_boxes[i]) / weights.sum()\n        overall_boxes.append(np.array(weighted_boxes))\n\n    overall_boxes = np.concatenate(overall_boxes, axis=0)\n    overall_boxes = overall_boxes[overall_boxes[:, 1].argsort()[::-1]]\n    boxes = overall_boxes[:, 2:]\n    scores = overall_boxes[:, 1]\n    labels = overall_boxes[:, 0]\n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:04:41.502801Z","iopub.execute_input":"2021-06-20T15:04:41.503993Z","iopub.status.idle":"2021-06-20T15:04:41.562262Z","shell.execute_reply.started":"2021-06-20T15:04:41.503947Z","shell.execute_reply":"2021-06-20T15:04:41.557328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_valid_transforms():\n    return A.Compose(\n        [\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:04:41.564212Z","iopub.execute_input":"2021-06-20T15:04:41.564571Z","iopub.status.idle":"2021-06-20T15:04:41.578713Z","shell.execute_reply.started":"2021-06-20T15:04:41.564531Z","shell.execute_reply":"2021-06-20T15:04:41.577721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"line_df = pd.read_csv(\"../input/line-detection-iam/forms.csv\")\nline_df[\"full_bb\"] = line_df.full_bb.apply(lambda x: literal_eval(x))\nline_df[\"line_bb\"] = line_df.line_bb.apply(lambda x: literal_eval(x))\nline_df_train = line_df[line_df[\"fold\"] != 0]\nTRAIN_ROOT_PATH = '../input/line-detection-iam/dataset/images/train'\n\nline_marking_train = []\nz = 0\nfor row in tqdm(line_df_train.values):\n    image_id = row[0]\n    \n    path = f'{TRAIN_ROOT_PATH}/{image_id}.jpg'\n    image = cv2.imread(path, cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0\n    \n    f = open(path.replace('images', 'labels').replace('jpg', 'txt'), 'r')\n    bbs = f.readlines()\n    bbs = np.array([[float(b) for b in bb.split(' ')] for bb in bbs])[:, 1:]\n    \n    bbs[:, 0] = bbs[:, 0] * image.shape[1]\n    bbs[:, 1] = bbs[:, 1] * image.shape[0]\n    bbs[:, 2] = bbs[:, 2] * image.shape[1]\n    bbs[:, 3] = bbs[:, 3] * image.shape[0]\n    bbs[:, 0] = bbs[:, 0] - (bbs[:, 2] / 2)\n    bbs[:, 1] = bbs[:, 1] - (bbs[:, 3] / 2)\n    bbs = bbs.astype(np.int32)\n    bbs[:, 0] = np.clip(bbs[:, 0], 0, image.shape[1])\n    bbs[:, 1] = np.clip(bbs[:, 1], 0, image.shape[0])\n    bbs[:, 2] = np.clip(bbs[:, 2], 0, image.shape[1])\n    bbs[:, 3] = np.clip(bbs[:, 3], 0, image.shape[0])\n    \n    f.close()\n    bbs = [bb.tolist() for bb in bbs]\n    \n    final = np.concatenate([np.array([[image_id]] * len(bbs)), bbs], axis=1).tolist()\n    line_marking_train += final\n    \n    z += 1\n    if z == 100:\n        pass\n\nline_marking_train = np.array(line_marking_train)\nline_marking_train = pd.DataFrame(line_marking_train, columns=['image_id', 'x', 'y', 'w', 'h'])\nline_marking_train","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:04:41.580579Z","iopub.execute_input":"2021-06-20T15:04:41.580976Z","iopub.status.idle":"2021-06-20T15:05:51.305533Z","shell.execute_reply.started":"2021-06-20T15:04:41.580935Z","shell.execute_reply":"2021-06-20T15:05:51.304653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"line_df = pd.read_csv(\"../input/line-detection-iam/forms.csv\")\nline_df[\"full_bb\"] = line_df.full_bb.apply(lambda x: literal_eval(x))\nline_df[\"line_bb\"] = line_df.line_bb.apply(lambda x: literal_eval(x))\nline_df_valid = line_df[line_df[\"fold\"] == 0]\nVALID_ROOT_PATH = '../input/line-detection-iam/dataset/images/valid'\n\nline_marking_valid = []\nz = 0\nfor row in tqdm(line_df_valid.values):\n    image_id = row[0]\n    \n    path = f'{VALID_ROOT_PATH}/{image_id}.jpg'\n    image = cv2.imread(path, cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0\n    \n    f = open(path.replace('images', 'labels').replace('jpg', 'txt'), 'r')\n    bbs = f.readlines()\n    bbs = np.array([[float(b) for b in bb.split(' ')] for bb in bbs])[:, 1:]\n    \n    bbs[:, 0] = bbs[:, 0] * image.shape[1]\n    bbs[:, 1] = bbs[:, 1] * image.shape[0]\n    bbs[:, 2] = bbs[:, 2] * image.shape[1]\n    bbs[:, 3] = bbs[:, 3] * image.shape[0]\n    bbs[:, 0] = bbs[:, 0] - (bbs[:, 2] / 2)\n    bbs[:, 1] = bbs[:, 1] - (bbs[:, 3] / 2)\n    bbs = bbs.astype(np.int32)\n    bbs[:, 0] = np.clip(bbs[:, 0], 0, image.shape[1])\n    bbs[:, 1] = np.clip(bbs[:, 1], 0, image.shape[0])\n    bbs[:, 2] = np.clip(bbs[:, 2], 0, image.shape[1])\n    bbs[:, 3] = np.clip(bbs[:, 3], 0, image.shape[0])\n    \n    f.close()\n    bbs = [bb.tolist() for bb in bbs]\n    \n    final = np.concatenate([np.array([[image_id]] * len(bbs)), bbs], axis=1).tolist()\n    line_marking_valid += final\n\nline_marking_valid = np.array(line_marking_valid)\nline_marking_valid = pd.DataFrame(line_marking_valid, columns=['image_id', 'x', 'y', 'w', 'h'])\nline_marking_valid","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:05:51.307165Z","iopub.execute_input":"2021-06-20T15:05:51.307523Z","iopub.status.idle":"2021-06-20T15:06:08.599421Z","shell.execute_reply.started":"2021-06-20T15:05:51.307484Z","shell.execute_reply":"2021-06-20T15:06:08.598572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        \n        if self.test or random.random() > 0.0:\n            image, boxes = self.load_image_and_boxes(index)\n        else:\n            image, boxes = self.load_cutmix_image_and_boxes(index)\n\n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        if self.test:\n            image = cv2.imread(f'{VALID_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        else:\n            image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values.astype(np.int32)\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return image, boxes\n\n    def load_cutmix_image_and_boxes(self, index, imsize=1024):\n        \"\"\" \n        This implementation of cutmix author:  https://www.kaggle.com/nvnnghia \n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize\n        s = imsize // 2\n    \n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y\n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]\n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32)\n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)\n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n            \n            boxes[:, 0] += padw\n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n\n        result_boxes = np.concatenate(result_boxes, 0)\n        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[np.where((result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1]) > 0)]\n        return result_image, result_boxes","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:06:08.601242Z","iopub.execute_input":"2021-06-20T15:06:08.601610Z","iopub.status.idle":"2021-06-20T15:06:08.642297Z","shell.execute_reply.started":"2021-06-20T15:06:08.601563Z","shell.execute_reply":"2021-06-20T15:06:08.641461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(line_marking_train.image_id.values)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:06:08.643711Z","iopub.execute_input":"2021-06-20T15:06:08.644087Z","iopub.status.idle":"2021-06-20T15:06:08.673735Z","shell.execute_reply.started":"2021-06-20T15:06:08.644050Z","shell.execute_reply":"2021-06-20T15:06:08.672922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_ROOT_PATH = '../input/line-detection-iam/dataset/images/train'\ndataset = DatasetRetriever(\n    image_ids=np.unique(line_marking_train.image_id.values),\n    marking=line_marking_train,\n    transforms=get_valid_transforms(),\n    test=False\n)\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndata_loader = DataLoader(\n    dataset,\n    batch_size=2,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:06:08.675013Z","iopub.execute_input":"2021-06-20T15:06:08.675382Z","iopub.status.idle":"2021-06-20T15:06:08.695747Z","shell.execute_reply.started":"2021-06-20T15:06:08.675346Z","shell.execute_reply":"2021-06-20T15:06:08.695111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VALID_ROOT_PATH = '../input/line-detection-iam/dataset/images/valid'\nvalid_dataset = DatasetRetriever(\n    image_ids=np.unique(line_marking_valid.image_id.values),\n    marking=line_marking_valid,\n    transforms=get_valid_transforms(),\n    test=True,\n)\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=2,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:06:08.697190Z","iopub.execute_input":"2021-06-20T15:06:08.697606Z","iopub.status.idle":"2021-06-20T15:06:08.708999Z","shell.execute_reply.started":"2021-06-20T15:06:08.697546Z","shell.execute_reply":"2021-06-20T15:06:08.708136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_net(checkpoint_path):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size=512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint['model_state_dict'])\n\n    del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\nnet = load_net('../input/efficientdetworking/effdet5-cutmix-augmix/EfficientDet-004-epoch.bin')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:06:08.710457Z","iopub.execute_input":"2021-06-20T15:06:08.710908Z","iopub.status.idle":"2021-06-20T15:06:20.592512Z","shell.execute_reply.started":"2021-06-20T15:06:08.710853Z","shell.execute_reply":"2021-06-20T15:06:20.591272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_predictions(images, score_threshold=0.22):\n    images = torch.stack(images).cuda().float()\n    predictions = []\n    with torch.no_grad():\n        det = net(images, torch.tensor([1]*images.shape[0]).float().cuda())\n        for i in range(images.shape[0]):\n            boxes = det[i].detach().cpu().numpy()[:,:4]    \n            scores = det[i].detach().cpu().numpy()[:,4]\n            indexes = np.where(scores > score_threshold)[0]\n            boxes = boxes[indexes]\n            boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n            predictions.append({\n                'boxes': boxes[indexes],\n                'scores': scores[indexes],\n            })\n    return [predictions]\n\ndef run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:06:20.594147Z","iopub.execute_input":"2021-06-20T15:06:20.594494Z","iopub.status.idle":"2021-06-20T15:06:20.619244Z","shell.execute_reply.started":"2021-06-20T15:06:20.594458Z","shell.execute_reply":"2021-06-20T15:06:20.618430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\ntrain_precisions = []\n\nfor k in tqdm(range(len(dataset))):\n    images, target, image_ids = dataset[k]\n    images_1, target, image_ids_1 = dataset[k]\n\n    images = [images, images_1]\n    image_ids = [image_ids, image_ids_1]\n\n    predictions = make_predictions(images, score_threshold=0.1)\n\n    i = 0\n    sample = images[i].permute(1,2,0).cpu().numpy()\n\n    boxes, scores, labels = run_wbf(predictions, image_index=i)\n    boxes = boxes.astype(np.int32).clip(min=0, max=511)\n\n    target['boxes'] = target['boxes'].numpy().astype(np.int32)\n    t = target['boxes'][:, 0].copy()\n    target['boxes'][:, 0] = target['boxes'][:, 1].copy()\n    target['boxes'][:, 1] = t\n    t = target['boxes'][:, 2].copy()\n    target['boxes'][:, 2] = target['boxes'][:, 3].copy()\n    target['boxes'][:, 3] = t\n\n    preds_sorted_idx = np.argsort(scores)[::-1]\n    boxes = boxes[preds_sorted_idx]\n\n    iou_thresholds = numba.typed.List()\n    for x in [0.5, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]:\n        iou_thresholds.append(x)\n    image_precision = calculate_image_precision(target['boxes'], boxes,\n                                                thresholds=iou_thresholds,\n                                                form='pascal_voc')\n    train_precisions.append(image_precision)\n#     print(image_precision)\n\nprint(sum(train_precisions) / len(train_precisions))\nprint(min(train_precisions), max(train_precisions))\nsns.distplot(train_precisions)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:15:39.741161Z","iopub.execute_input":"2021-06-20T15:15:39.741536Z","iopub.status.idle":"2021-06-20T15:15:43.666478Z","shell.execute_reply.started":"2021-06-20T15:15:39.741503Z","shell.execute_reply":"2021-06-20T15:15:43.664575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\ntrain_precisions = []\n\nfor k in tqdm(range(len(valid_dataset))):    \n    images, target, image_ids = valid_dataset[k]\n    images_1, target, image_ids_1 = valid_dataset[k]\n\n    images = [images, images_1]\n    image_ids = [image_ids, image_ids_1]\n\n    predictions = make_predictions(images, score_threshold=0.1)\n\n    i = 0\n    sample = images[i].permute(1,2,0).cpu().numpy()\n\n    boxes, scores, labels = run_wbf(predictions, image_index=i)\n    boxes = boxes.astype(np.int32).clip(min=0, max=511)\n\n    target['boxes'] = target['boxes'].numpy().astype(np.int32)\n    t = target['boxes'][:, 0].copy()\n    target['boxes'][:, 0] = target['boxes'][:, 1].copy()\n    target['boxes'][:, 1] = t\n    t = target['boxes'][:, 2].copy()\n    target['boxes'][:, 2] = target['boxes'][:, 3].copy()\n    target['boxes'][:, 3] = t\n\n    preds_sorted_idx = np.argsort(scores)[::-1]\n    boxes = boxes[preds_sorted_idx]\n\n    iou_thresholds = numba.typed.List()\n    for x in [0.5, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]:\n        iou_thresholds.append(x)\n    image_precision = calculate_image_precision(target['boxes'], boxes,\n                                                thresholds=iou_thresholds,\n                                                form='pascal_voc')\n    train_precisions.append(image_precision)\n#     print(image_precision)\n\nprint(sum(train_precisions) / len(train_precisions))\nprint(min(train_precisions), max(train_precisions))\nsns.distplot(train_precisions)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:19:22.309009Z","iopub.execute_input":"2021-06-20T15:19:22.309351Z","iopub.status.idle":"2021-06-20T15:19:24.086315Z","shell.execute_reply.started":"2021-06-20T15:19:22.309320Z","shell.execute_reply":"2021-06-20T15:19:24.084464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\n\nfor k in range(10, 20):\n    images, target, image_ids = dataset[k]\n    images_1, target, image_ids_1 = dataset[k]\n    \n#     images, target, image_ids = valid_dataset[k]\n#     images_1, target, image_ids_1 = valid_dataset[k]\n\n    images = [images, images_1]\n    image_ids = [image_ids, image_ids_1]\n\n    predictions = make_predictions(images, score_threshold=0.1)\n\n    i = 0\n    sample = images[i].permute(1,2,0).cpu().numpy()\n\n    boxes, scores, labels = run_wbf(predictions, image_index=i)\n    boxes = boxes.astype(np.int32).clip(min=0, max=511)\n\n    target['boxes'] = target['boxes'].numpy().astype(np.int32)\n    t = target['boxes'][:, 0].copy()\n    target['boxes'][:, 0] = target['boxes'][:, 1].copy()\n    target['boxes'][:, 1] = t\n    t = target['boxes'][:, 2].copy()\n    target['boxes'][:, 2] = target['boxes'][:, 3].copy()\n    target['boxes'][:, 3] = t\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for box in target['boxes']:\n        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (0, 1, 0), 3)\n\n    for box in boxes:\n        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (1, 0, 0), 3)\n    preds_sorted_idx = np.argsort(scores)[::-1]\n    boxes = boxes[preds_sorted_idx]\n\n    iou_thresholds = numba.typed.List()\n    for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n        iou_thresholds.append(x)\n    image_precision = calculate_image_precision(target['boxes'], boxes,\n                                                thresholds=iou_thresholds,\n                                                form='pascal_voc')\n\n    ax.set_axis_off()\n    ax.imshow(sample);","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:21:40.732189Z","iopub.execute_input":"2021-06-20T15:21:40.732515Z","iopub.status.idle":"2021-06-20T15:21:44.407216Z","shell.execute_reply.started":"2021-06-20T15:21:40.732483Z","shell.execute_reply":"2021-06-20T15:21:44.406159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\n\nfor k in range(10, 20):\n#     images, target, image_ids = dataset[k]\n#     images_1, target, image_ids_1 = dataset[k]\n    \n    images, target, image_ids = valid_dataset[k]\n    images_1, target, image_ids_1 = valid_dataset[k]\n\n    images = [images, images_1]\n    image_ids = [image_ids, image_ids_1]\n\n    predictions = make_predictions(images, score_threshold=0.1)\n\n    i = 0\n    sample = images[i].permute(1,2,0).cpu().numpy()\n\n    boxes, scores, labels = run_wbf(predictions, image_index=i)\n    boxes = boxes.astype(np.int32).clip(min=0, max=511)\n\n    target['boxes'] = target['boxes'].numpy().astype(np.int32)\n    t = target['boxes'][:, 0].copy()\n    target['boxes'][:, 0] = target['boxes'][:, 1].copy()\n    target['boxes'][:, 1] = t\n    t = target['boxes'][:, 2].copy()\n    target['boxes'][:, 2] = target['boxes'][:, 3].copy()\n    target['boxes'][:, 3] = t\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for box in target['boxes']:\n        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (0, 1, 0), 3)\n\n    for box in boxes:\n        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (1, 0, 0), 3)\n    preds_sorted_idx = np.argsort(scores)[::-1]\n    boxes = boxes[preds_sorted_idx]\n\n    iou_thresholds = numba.typed.List()\n    for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n        iou_thresholds.append(x)\n    image_precision = calculate_image_precision(target['boxes'], boxes,\n                                                thresholds=iou_thresholds,\n                                                form='pascal_voc')\n\n    ax.set_axis_off()\n    ax.imshow(sample);","metadata":{"execution":{"iopub.status.busy":"2021-06-20T15:21:10.698814Z","iopub.execute_input":"2021-06-20T15:21:10.699153Z","iopub.status.idle":"2021-06-20T15:21:14.323149Z","shell.execute_reply.started":"2021-06-20T15:21:10.699122Z","shell.execute_reply":"2021-06-20T15:21:14.322107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}