{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np \nimport cv2\nimport random\nfrom PIL import Image\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset\nimport torch.nn.functional as F\nfrom albumentations import *\n\nfrom tqdm import tqdm\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nfrom torchvision.models.detection.backbone_utils import resnet_fpn_backbone\nfrom torchvision.models.detection.faster_rcnn import FasterRCNN\nfrom torchvision.models.detection.backbone_utils import BackboneWithFPN\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:31.828058Z","iopub.execute_input":"2021-12-10T06:41:31.828375Z","iopub.status.idle":"2021-12-10T06:41:32.965406Z","shell.execute_reply.started":"2021-12-10T06:41:31.828326Z","shell.execute_reply":"2021-12-10T06:41:32.964585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_aug(aug):\n    return Compose(aug, bbox_params=BboxParams(format='pascal_voc', min_area=0, min_visibility=0, label_fields=['category_id']))\n\ndef bb_overlap(boxA, boxB):\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)\n    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)\n    iou = interArea / float(boxAArea)\n    return iou\n\nclass WheatDataset(Dataset):\n    def __init__(self, df, img_size, mode='train', network='FasterRCNN', bbox_removal_threshold=0.25):\n        super(WheatDataset,self).__init__()\n        self.df = df\n        self.image_ids = list(np.unique(self.df.image_id.values))\n        self.img_size = img_size\n        self.root_dir = '../input/global-wheat-detection/train'\n        self.w2017_ext_dir = 'dataset/wheat2017'\n        self.spike_ext_dir = 'dataset/spike-wheat'\n        assert mode in  ['train', 'valid']\n        self.mode = mode\n        assert network in ['FasterRCNN', 'EffDet']\n        self.network = network\n        self.bbox_removal_threshold = bbox_removal_threshold\n        if self.mode == 'train':\n            random.shuffle(self.image_ids)\n        self.train_transforms = get_aug([\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ToGray(p=0.01),\n            GaussNoise(p=0.2),\n            OneOf([\n                MotionBlur(p=0.2),\n                MedianBlur(blur_limit=3, p=0.1),\n                Blur(blur_limit=3, p=0.1),\n            ], p=0.2),\n            RandomBrightnessContrast(p=0.25),\n#             HueSaturationValue(p=0.25)\n        ])\n        self.resize_transforms = get_aug([\n            Resize(height=self.img_size, width=self.img_size, interpolation=1, p=1)\n        ])\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def refine_boxes(self, boxes):\n        result_boxes = []\n        for box in boxes:\n            if box[2] - box[0] < 10 or box[3] - box[1] < 10:\n                continue\n            result_boxes.append(box)\n        result_boxes = np.array(result_boxes)\n        return result_boxes\n\n    def resize_image(self, image, boxes):\n        cats = np.ones(boxes.shape[0], dtype=int)\n        annotations = {'image': image, 'bboxes': boxes, 'category_id': cats}\n        augmented = self.resize_transforms(**annotations)\n        image = augmented['image']\n        boxes = np.array(augmented['bboxes'])\n        return image, boxes\n\n    def crop_image(self, image, boxes, xmin, ymin, xmax, ymax):\n        image = image[ymin:ymax,xmin:xmax,:]\n        cutout_box = [xmin, ymin, xmax, ymax]\n        result_boxes = []\n        for box in boxes:\n            iou = bb_overlap(box, cutout_box)\n            if iou > self.bbox_removal_threshold:\n                result_boxes.append(box)\n        if len(result_boxes) > 0:\n            result_boxes = np.array(result_boxes, dtype=float)\n            result_boxes[:,[0,2]] -= xmin\n            result_boxes[:,[1,3]] -= ymin\n            result_boxes[:,[0,2]] = result_boxes[:,[0,2]].clip(0, xmax-xmin)\n            result_boxes[:,[1,3]] = result_boxes[:,[1,3]].clip(0, ymax-ymin)\n        else:\n            result_boxes = np.array([], dtype=float).reshape(0,4)\n        return image, result_boxes\n    \n    def random_crop_resize(self, image, boxes, img_size=1024, p=0.5):\n        if random.random() > p:\n            new_img_size = random.randint(int(0.75*img_size), img_size)\n            x = random.randint(0, img_size-new_img_size)\n            y = random.randint(0, img_size-new_img_size)\n            image, boxes = self.crop_image(image, boxes, x, y, x+new_img_size, y+new_img_size)\n            return self.resize_image(image, boxes)\n        else:\n            if self.img_size != 1024:\n                return self.resize_image(image, boxes)\n            else:\n                return image, boxes\n\n    def load_image_and_boxes(self, image_id):\n        tmp_df = self.df.loc[self.df['image_id']==image_id]\n        source = np.unique(tmp_df.source.values)[0]\n        if source == 'wheat2017':\n            img_path = '{}/{}.jpg'.format(self.w2017_ext_dir, image_id)\n        elif source == 'spike':\n            img_path = '{}/{}.jpg'.format(self.spike_ext_dir, image_id)\n        else:\n            img_path = '{}/{}.jpg'.format(self.root_dir, image_id)\n        \n        img = Image.open(img_path)\n        img = img.convert('RGB')\n        img = np.array(img, dtype=np.uint8)\n\n        boxes = []\n        for _, row in tmp_df.iterrows():\n            if row['isbox'] == False:\n                continue\n            boxes.append([float(row['xmin']),float(row['ymin']),float(row['xmax']),float(row['ymax'])])\n\n        boxes = self.refine_boxes(boxes)\n        if len(boxes) > 0:\n            boxes = np.array(boxes, dtype=float)\n        else:\n            boxes = np.array([], dtype=float).reshape(0,4)\n        return img, boxes, source\n    \n    def load_cutmix_image_and_boxes(self, image_id, imsize=1024):   #custom mosaic data augmentation\n        image_ids = self.image_ids.copy()\n        image_ids.remove(image_id)\n        cutmix_image_ids = [image_id] + random.sample(image_ids, 3)\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.uint8)\n        result_boxes = []\n        \n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]\n        for i, img_id in enumerate(cutmix_image_ids):\n            image, boxes, source = self.load_image_and_boxes(img_id)\n            if source == 'spike':\n                height, width = image.shape[0:2]\n                if i == 0 or i == 3:\n                    image, boxes = self.crop_image(image, boxes, xmin=width-1024, ymin=0, xmax=width, ymax=1024)\n                else:\n                    image, boxes = self.crop_image(image, boxes, xmin=0, ymin=0, xmax=1024, ymax=1024)\n            if i == 0:\n                image, boxes = self.crop_image(image, boxes, imsize-xc, imsize-yc, imsize, imsize)\n                result_image[0:yc, 0:xc,:] = image\n                result_boxes.extend(boxes)\n            elif i == 1:\n                image, boxes = self.crop_image(image, boxes, 0, imsize-yc, imsize-xc, imsize)\n                result_image[0:yc, xc:imsize, :] = image\n                if boxes.shape[0] > 0:\n                    boxes[:,[0,2]] += xc\n                result_boxes.extend(boxes)\n            elif i == 2:\n                image, boxes = self.crop_image(image, boxes, 0, 0, imsize-xc, imsize-yc)\n                result_image[yc:imsize, xc:imsize, :] = image\n                if boxes.shape[0] > 0:\n                    boxes[:,[0,2]] += xc\n                    boxes[:,[1,3]] += yc\n                result_boxes.extend(boxes)\n            else:\n                image, boxes = self.crop_image(image, boxes, imsize-xc, 0, imsize, imsize-yc)\n                result_image[yc:imsize, 0:xc, :] = image\n                if boxes.shape[0] > 0:\n                    boxes[:,[1,3]] += yc\n                result_boxes.extend(boxes)\n            del image\n            del boxes\n        del cutmix_image_ids\n        del image_ids\n        if len(result_boxes) == 0:\n            result_boxes = np.array([], dtype=float).reshape(0,4)\n        else:\n            result_boxes = np.vstack(result_boxes)\n            result_boxes[:,[0,2]] = result_boxes[:,[0,2]].clip(0, imsize)\n            result_boxes[:,[1,3]] = result_boxes[:,[1,3]].clip(0, imsize)\n        return result_image, result_boxes\n    \n    def __getitem__(self, index):\n        image_id = self.image_ids[index]\n        if self.mode == 'train':\n            while(True):\n                if random.random() > 0.5:\n                    image, boxes, source = self.load_image_and_boxes(image_id)\n                    if source == 'spike':\n                        height, width = image.shape[0:2]\n                        if random.random() > 0.5:\n                            image, boxes = self.crop_image(image, boxes, xmin=0, ymin=0, xmax=1024, ymax=1024)\n                        else:\n                            image, boxes = self.crop_image(image, boxes, xmin=width-1024, ymin=0, xmax=width, ymax=1024)\n                else:\n                    image, boxes = self.load_cutmix_image_and_boxes(image_id)\n\n                image, boxes = self.random_crop_resize(image, boxes, p=0.5)\n                if len(boxes) > 0:\n                    cats = np.ones(boxes.shape[0], dtype=int)\n                    annotations = {'image': image, 'bboxes': boxes, 'category_id': cats}\n                    augmented = self.train_transforms(**annotations)\n                    image = augmented['image']\n                    boxes = np.array(augmented['bboxes'])\n                    break\n        else:\n            image, boxes, _ = self.load_image_and_boxes(image_id)\n            if self.img_size != 1024:\n                image, boxes = self.resize_image(image, boxes)\n\n        if self.network == 'EffDet':\n            if boxes.shape[0] == 0:\n                target = {\n                    \"boxes\": torch.zeros((0, 4), dtype=torch.float32),\n                    \"labels\": torch.zeros(0, dtype=torch.int64)\n                }\n            else:\n                boxes[:,[0,1,2,3]] = boxes[:,[1,0,3,2]]\n                target = {\n                    'boxes': torch.as_tensor(boxes, dtype=torch.float32),\n                    'labels': torch.ones((boxes.shape[0],), dtype=torch.int64)\n                }\n        else:\n            if boxes.shape[0] == 0:\n                target = {\n                    \"boxes\": torch.zeros((0, 4), dtype=torch.float32),\n                    \"labels\": torch.zeros(0, dtype=torch.int64)\n#                     \"area\": torch.zeros(0, dtype=torch.float32),\n#                     \"iscrowd\": torch.zeros((0,), dtype=torch.int64)\n                }\n            else:\n                target = {}\n                area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n                target['boxes'] = torch.as_tensor(boxes, dtype=torch.float32)\n                target['labels'] = torch.ones((boxes.shape[0],), dtype=torch.int64)\n#                 target['area'] = torch.as_tensor(area, dtype=torch.float32)\n#                 target['iscrowd'] = torch.zeros((boxes.shape[0],), dtype=torch.int64)\n            \n        image = image.astype(np.float32)\n        image /= 255.0\n        image = torch.from_numpy(image).permute(2,0,1)\n        return image, target","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:32.970844Z","iopub.execute_input":"2021-12-10T06:41:32.97111Z","iopub.status.idle":"2021-12-10T06:41:33.138281Z","shell.execute_reply.started":"2021-12-10T06:41:32.971074Z","shell.execute_reply":"2021-12-10T06:41:33.137564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.139575Z","iopub.execute_input":"2021-12-10T06:41:33.14006Z","iopub.status.idle":"2021-12-10T06:41:33.147081Z","shell.execute_reply.started":"2021-12-10T06:41:33.140023Z","shell.execute_reply":"2021-12-10T06:41:33.14638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/wheat-src/csv/gwd2020.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.149875Z","iopub.execute_input":"2021-12-10T06:41:33.150197Z","iopub.status.idle":"2021-12-10T06:41:33.267844Z","shell.execute_reply.started":"2021-12-10T06:41:33.150162Z","shell.execute_reply":"2021-12-10T06:41:33.267063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 3\nvalid_df = df.loc[df['fold'] == fold]\ntrain_df = df.loc[~df.index.isin(valid_df.index)]\nvalid_df = valid_df.loc[valid_df['isbox']==True].reset_index(drop=True)\n# fold = 1\n# train_df = df.loc[df['isbox']==True].reset_index(drop=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.269183Z","iopub.execute_input":"2021-12-10T06:41:33.269811Z","iopub.status.idle":"2021-12-10T06:41:33.287882Z","shell.execute_reply.started":"2021-12-10T06:41:33.269758Z","shell.execute_reply":"2021-12-10T06:41:33.287194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 1024\ntrain_dataset = WheatDataset(df=train_df, img_size=img_size, mode='train', network='FasterRCNN')\nvalid_dataset = WheatDataset(df=valid_df, img_size=img_size, mode='valid', network='FasterRCNN')","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.289276Z","iopub.execute_input":"2021-12-10T06:41:33.289795Z","iopub.status.idle":"2021-12-10T06:41:33.451564Z","shell.execute_reply.started":"2021-12-10T06:41:33.289732Z","shell.execute_reply":"2021-12-10T06:41:33.450819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 8\nnum_workers = 2\ntrain_loader = DataLoader(\n            train_dataset,\n            batch_size=batch_size,\n            sampler=RandomSampler(train_dataset),\n            pin_memory=False,\n            drop_last=False,\n            num_workers=num_workers,\n            collate_fn=collate_fn\n        )\n\nvalid_loader = DataLoader(\n            valid_dataset,\n            batch_size=batch_size,\n            shuffle=False,\n            sampler=SequentialSampler(valid_dataset),\n            pin_memory=False,\n            num_workers=num_workers,\n            collate_fn=collate_fn\n        )","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.452961Z","iopub.execute_input":"2021-12-10T06:41:33.453226Z","iopub.status.idle":"2021-12-10T06:41:33.45898Z","shell.execute_reply.started":"2021-12-10T06:41:33.453191Z","shell.execute_reply":"2021-12-10T06:41:33.458296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# temp = iter(train_loader)\n# images, labels = next(iter(temp))","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.460293Z","iopub.execute_input":"2021-12-10T06:41:33.460717Z","iopub.status.idle":"2021-12-10T06:41:33.467615Z","shell.execute_reply.started":"2021-12-10T06:41:33.460681Z","shell.execute_reply":"2021-12-10T06:41:33.466952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fasterrcnn_resnet_fpn(backbone_name='resnet152', progress=True, num_classes=91, pretrained=True, pretrained_backbone=True, **kwargs):\n    if backbone_name == 'resnet50':\n        model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=pretrained)\n    elif backbone_name in ['resnet101', 'resnet152']:\n        backbone = resnet_fpn_backbone(backbone_name, pretrained_backbone)\n    else:\n        backbone = my_resnet_fpn_backbone(backbone_name, pretrained_backbone)\n    model = FasterRCNN(backbone, num_classes, **kwargs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.468977Z","iopub.execute_input":"2021-12-10T06:41:33.469232Z","iopub.status.idle":"2021-12-10T06:41:33.476138Z","shell.execute_reply.started":"2021-12-10T06:41:33.469197Z","shell.execute_reply":"2021-12-10T06:41:33.475313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.478343Z","iopub.execute_input":"2021-12-10T06:41:33.479112Z","iopub.status.idle":"2021-12-10T06:41:33.485013Z","shell.execute_reply.started":"2021-12-10T06:41:33.479075Z","shell.execute_reply":"2021-12-10T06:41:33.484278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True, progress=False)\nin_features = model.roi_heads.box_predictor.cls_score.in_features\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, 2)\nif torch.cuda.is_available():\n    model = model.cuda()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:33.486377Z","iopub.execute_input":"2021-12-10T06:41:33.486633Z","iopub.status.idle":"2021-12-10T06:41:36.993989Z","shell.execute_reply.started":"2021-12-10T06:41:33.486601Z","shell.execute_reply":"2021-12-10T06:41:36.9932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = fasterrcnn_resnet_fpn(backbone_name='resnet50', pretrained=True, pretrained_backbone=True)\n# in_features = model.roi_heads.box_predictor.cls_score.in_features\n# model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 2)\n# if torch.cuda.is_available():\n#     model = model.cuda()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:36.995269Z","iopub.execute_input":"2021-12-10T06:41:36.995538Z","iopub.status.idle":"2021-12-10T06:41:37.000366Z","shell.execute_reply.started":"2021-12-10T06:41:36.995504Z","shell.execute_reply":"2021-12-10T06:41:36.999671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.0005, momentum=0.9, weight_decay=0.0005)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:37.003454Z","iopub.execute_input":"2021-12-10T06:41:37.003931Z","iopub.status.idle":"2021-12-10T06:41:37.012323Z","shell.execute_reply.started":"2021-12-10T06:41:37.003887Z","shell.execute_reply":"2021-12-10T06:41:37.011592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport numba\nimport re\nimport cv2\nimport ast\nimport matplotlib.pyplot as plt\n\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    if tp == 0 and (tp + fp + fn) == 0:\n        return 1.0\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision\n\n# Numba typed list!\niou_thresholds = numba.typed.List()\nfor x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n    iou_thresholds.append(x)\n\ndef calculate_final_score(all_predictions, score_threshold):\n    final_scores = []\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        pred_boxes = all_predictions[i]['pred_boxes'].copy()\n        scores = all_predictions[i]['scores'].copy()\n\n        indexes = np.where(scores>score_threshold)\n        pred_boxes = pred_boxes[indexes]\n        scores = scores[indexes]\n\n        image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')\n        final_scores.append(image_precision)\n\n    return np.mean(final_scores)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:37.014966Z","iopub.execute_input":"2021-12-10T06:41:37.015256Z","iopub.status.idle":"2021-12-10T06:41:38.026669Z","shell.execute_reply.started":"2021-12-10T06:41:37.015224Z","shell.execute_reply":"2021-12-10T06:41:38.025939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 20\nap_max = 0\n\nloss_hist = AverageMeter()\nfor epoch in range(epochs):\n    loss_hist.reset()\n    model.train()\n\n\n    loop = tqdm(train_loader)\n    for images, targets in loop:\n        ### mixup\n        if random.random() > 0.5:\n            images = torch.stack(images).cuda()\n            shuffle_indices = torch.randperm(images.size(0))\n            indices = torch.arange(images.size(0))\n            lam = np.clip(np.random.beta(1.0, 1.0), 0.35, 0.65)\n            images = lam * images + (1 - lam) * images[shuffle_indices, :]\n            mix_targets = []\n            for i, si in zip(indices, shuffle_indices):\n                if i.item() == si.item():\n                    target = targets[i.item()]\n                else:\n                    target = {\n                        'boxes': torch.cat([targets[i.item()]['boxes'], targets[si.item()]['boxes']]),\n                        'labels': torch.cat([targets[i.item()]['labels'], targets[si.item()]['labels']])\n#                         'area': torch.cat([targets[i.item()]['area'], targets[si.item()]['area']]),\n#                         'iscrowd': torch.cat([targets[i.item()]['iscrowd'], targets[si.item()]['iscrowd']])\n                    }\n\n                mix_targets.append(target)\n            targets = mix_targets\n        if torch.cuda.is_available():    \n            images = list(image.cuda() for image in images)\n            targets = [{k: v.cuda() for k, v in t.items()} for t in targets]\n\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):\n            loss_dict = model(images, targets)\n            losses = sum(loss for loss in loss_dict.values())\n            if losses == 0 or not torch.isfinite(losses):\n                continue\n            losses.backward()\n            optimizer.step()\n            loss_hist.update(losses.detach().item(), len(images))\n    \n    train_loss = loss_hist.avg\n    print(f'Epoch {epoch} train_loss {train_loss}')\n#     if epoch > 40:\n# #         torch.save(model.state_dict(), f'./model_ep-{epoch}_trainloss-{train_loss}')\n#         torch.save(model, f'./model_ep-{epoch}_trainloss-{train_loss}')\n    model.eval()\n    all_predictions = []\n    for images, targets in tqdm(valid_loader):\n        images = list(image.cuda() for image in images)\n        with torch.set_grad_enabled(False):\n            outputs = model(images)\n        for t, o in zip(targets, outputs):\n            boxes = o['boxes'].data.cpu().numpy()\n            boxes = boxes.clip(min=0, max=1024-1)\n            scores = o['scores'].data.cpu().numpy()\n            all_predictions.append({\n                'pred_boxes': boxes.astype(int),\n                'scores': scores,\n                'gt_boxes': t['boxes'].cpu().numpy().astype(int)\n            })\n    ap = calculate_final_score(all_predictions, score_threshold=0.5)\n    print('Epoch {} Train loss: {:.5f} | Val AP: {:.5f}'.format(epoch, train_loss, ap))\n\n    if ap > ap_max:\n        ap_max = ap\n        torch.save(model, f'./model_ep-{epoch}_ap-{ap}')","metadata":{"execution":{"iopub.status.busy":"2021-12-10T06:41:38.030067Z","iopub.execute_input":"2021-12-10T06:41:38.030265Z","iopub.status.idle":"2021-12-10T09:45:34.542097Z","shell.execute_reply.started":"2021-12-10T06:41:38.030241Z","shell.execute_reply":"2021-12-10T09:45:34.541317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), f'./modeldic_ep-{epoch}_trainloss-{train_loss}')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T09:45:34.546464Z","iopub.execute_input":"2021-12-10T09:45:34.548694Z","iopub.status.idle":"2021-12-10T09:45:34.881344Z","shell.execute_reply.started":"2021-12-10T09:45:34.548625Z","shell.execute_reply":"2021-12-10T09:45:34.880598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}