{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch starter - FasterRCNN Inference\n\n- You can find the [train notebook here](https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train)\n- The weights are [available here](https://www.kaggle.com/dataset/7d5f1ed9454c848ecb909c109c6fa8e573ea4de299e249c79edc6f47660bf4c5)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\nfrom ensemble_boxes import *\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval\nfrom effdet.efficientdet import HeadNet\n\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport os\nimport re\nimport gc\n\nfrom PIL import Image\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport torch\nimport torchvision\n\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\n\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\n\nfrom matplotlib import pyplot as plt\n\n\nDIR_INPUT = '/kaggle/input/global-wheat-detection'\nDIR_TRAIN = f'{DIR_INPUT}/train'\nDIR_TEST = f'{DIR_INPUT}/test'\n\n\n#'/kaggle/input/wwdadamfasterrcnn'\n#'/kaggle/input/fasterrcnn-wheat-detection60epoch'\n#'/kaggle/input/fasterrcnn-resnet50-fpn-wheat-detection'\n#'/kaggle/input/global-wheat-detection-public'\n# '/kaggle/input/gwd-fasterrcnn-30-epoch-good'\n\nDIR_MAIN = '/kaggle/input'\nDIR_WEIGHTS = '/kaggle/input/gwd-fasterrcnn-20-epoch-mosaic'\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\n#fasterrcnn_resnet50_fpn.pth\n#fasterrcnn_resnet50_fpn_heavy_aug.pth\n#fasterrcnn_resnet50_fpn_10_epoch_mosaic.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\ndef load_efficient_net(checkpoint_path):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size=512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(checkpoint_path)\n\n    checkpoint2 = OrderedDict()\n\n    for v in checkpoint:\n        if v.startswith('anchors.boxes'):\n            continue\n            \n        checkpoint2[v[6:]] = checkpoint[v]\n\n    \n    net.load_state_dict(checkpoint2)\n\n    del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\ndef load_net(checkpoint_path):\n    net = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=False)\n    num_classes = 2  # 1 class (wheat) + background\n    # get number of input features for the classifier\n    in_features = net.roi_heads.box_predictor.cls_score.in_features\n    # replace the pre-trained head with a new one\n    net.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint)\n    net = net.cuda()\n    net.eval()\n\n    del checkpoint\n    gc.collect()\n    return net\n\n\nmodels = [\n#         load_net(f'{DIR_MAIN}/fasterrcnn-resnet50-fpn-wheat-detection/fasterrcnn_resnet50_fpn.pth'), #TEST: 0.6643\n#         load_net(f'{DIR_MAIN}/wwd-sgd-heavy-aug/fasterrcnn_resnet50_fpn_heavy_aug.pth'), #TEST: 0.6623\n# #         load_net(f'{DIR_MAIN}/gwd-fasterrcnn-10-epoch-mosaic/fasterrcnn_resnet50_fpn_10_epoch_mosaic.pth'), # TEST: 0.6570\n#         load_net(f'{DIR_MAIN}/gwd-fasterccn-30e-mosaic-heavy/fasterrcnn_resnet50_fpn_30e_mosaic_heavy.pth') # TEST: 0.6606\n   load_net(f'{DIR_MAIN}/gwd-fasterrcnn-resnet50-fpn-fold0/fasterrcnn_resnet50_fpn_fold0.pth'),\n   load_net(f'{DIR_MAIN}/gwd-fasterrcnn-resnet50-fpn-fold1/fasterrcnn_resnet50_fpn_fold1.pth'),\n   load_net(f'{DIR_MAIN}/gwd-fasterrcnn-resnet50-fpn-fold2/fasterrcnn_resnet50_fpn_fold2.pth'),\n   load_net(f'{DIR_MAIN}/gwd-fasterrcnn-resnet50-fpn-fold3/fasterrcnn_resnet50_fpn_fold3.pth'),\n   load_net(f'{DIR_MAIN}/gwd-fasterrcnn-resnet50-fpn-fold4/fasterrcnn_resnet50_fpn_fold4.pth')\n]\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(f'{DIR_INPUT}/sample_submission.csv')\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = 512\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import product\n\ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WheatTestDataset(Dataset):\n\n    def __init__(self, dataframe, image_dir, transforms=None):\n        super().__init__()\n\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        self.image_mean = [0.43216, 0.394666, 0.37645]      \n        self.image_std = [0.22803, 0.22145, 0.216989]\n        \n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)#.astype(np.float32)\n#         image /= 255.0\n\n        if self.transforms:\n            sample = {\n                'image': image,\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']/255.0 #self.normalize(sample['image']/255.0)\n\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n    \n    def normalize(self, image):\n        dtype, device = image.dtype, image.device\n        mean = torch.as_tensor(self.image_mean, dtype=dtype, device=device)\n        std = torch.as_tensor(self.image_std, dtype=dtype, device=device)\n        return (image - mean[:, None, None]) / std[:, None, None]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Albumentations\ndef get_test_transform():\n    return A.Compose([\n        A.Resize(512, 512),\n        ToTensorV2(p=1.0)\n    ])\n\n# Albumentations\ndef get_flip_test_transform():\n    return A.Compose([\n        A.VerticalFlip(p=1.0),\n        ToTensorV2(p=1.0)\n    ])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntest_dataset = WheatTestDataset(test_df, DIR_TEST, get_test_transform())\n\ntest_data_loader = DataLoader(\n    test_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_predictions(images, score_threshold=0.1):\n    images = torch.stack(images).cuda().float()\n    predictions = []\n    for net in models:\n        with torch.no_grad():\n            det = net(images, torch.tensor([1]*images.shape[0]).float().cuda())\n            for i in range(images.shape[0]):\n                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                scores = det[i].detach().cpu().numpy()[:,4]\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                predictions.append({\n                    'boxes': boxes[indexes],\n                    'scores': scores[indexes],\n                })\n    return [predictions]\n\ndef make_tta_predictions(images, score_threshold=0.5):\n    images = torch.stack(images).float().cuda()\n    predictions = []\n    for net in models:\n        with torch.no_grad():                      \n            for tta_transform in tta_transforms:\n                result = []\n                det = net(tta_transform.batch_augment(images.clone()))\n                print(det)\n                for i in range(images.shape[0]):\n                    boxes = det[i].detach().cpu().numpy()[:,:4]    \n                    scores = det[i].detach().cpu().numpy()[:,4]\n                    indexes = np.where(scores > score_threshold)[0]\n                    boxes = boxes[indexes]\n                    boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                    boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                    boxes = tta_transform.deaugment_boxes(boxes.copy())\n                    result.append({\n                        'boxes': boxes,\n                        'scores': scores[indexes],\n                    })\n                predictions.append(result)\n    return predictions\n\ndef make_tta_ensemble_predictions(images, score_threshold=0.5):\n    images = torch.stack(images).float().cuda()\n    predictions = []\n    for net in models:\n        with torch.no_grad():                      \n            for tta_transform in tta_transforms:\n                result = []\n                det = net(tta_transform.batch_augment(images.clone()))\n\n                for i in range(images.shape[0]):\n                    boxes = det[i]['boxes'].detach().cpu().numpy()  \n                    scores = det[i]['scores'].detach().cpu().numpy()\n                    indexes = np.where(scores > score_threshold)[0]\n                    boxes = boxes[indexes]\n#                     boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n#                     boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                    boxes = tta_transform.deaugment_boxes(boxes.copy())\n                    result.append({\n                        'boxes': boxes,\n                        'scores': scores[indexes],\n                    })\n                predictions.append(result)\n    return predictions\n\ndef make_ensemble_predictions(images):\n#     transform = A.VerticalFlip(p=1.0)\n#     images_lr = [torch.as_tensor(transform.apply(image.numpy())) for image in images]\n    \n    images = list(image.to(device) for image in images)\n#     images_lr = list(image.to(device) for image in images_lr)\n    \n    result = []\n    for net in models:\n        outputs = net(images)\n        result.append(outputs)\n        \n#         images = [np.fliplr(image) for image in images]\n#         outputs = net(images_lr)\n#         for output in outputs:\n#             outputs['boxes'] = [transform.apply_to_bbox(box) for box in output['boxes']]\n#         outputs[:]['boxes'] = map(transform.apply_to_bbox, outputs[:]['boxes'])\n    \n#     for net in models:\n#         images = [np.fliplr(image) for image in images]\n#         outputs = net(images)\n#         outputs['boxes'] = map(np.fliplr, outputs['boxes'])\n#         result.append(outputs)\n#     print(outputs[0]['boxes'], type(outputs[0]['boxes']))\n    return result\n\n# from ensemble_boxes import *\n# def run_wbf(predictions, image_index, image_size=512, iou_thr=0.55, skip_box_thr=0.7, weights=None):\n#     boxes = [prediction[image_index]['boxes'].data.cpu().numpy()/(image_size-1) for prediction in predictions]\n#     scores = [prediction[image_index]['scores'].data.cpu().numpy() for prediction in predictions]\n#     labels = [np.ones(prediction[image_index]['scores'].shape[0]) for prediction in predictions]\n#     boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n#     boxes = boxes*(image_size-1)\n#     return boxes, scores, labels\n\ndef run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_threshold = 0.5\nresults = []\n\nfor images, image_ids in test_data_loader:\n\n    images = list(image.to(device) for image in images)\n#     outputs = model(images)\n#     outputs = make_ensemble_predictions(images)\n    outputs = make_tta_ensemble_predictions(images)\n\n    for i, image in enumerate(images):\n#         boxes, scores, labels = run_wbf([outputs],image_size=1024, image_index=i)\n#         boxes = (boxes*2).astype(np.int32).clip(min=0, max=1023)\n\n        boxes, scores, labels = run_wbf(outputs, image_index=i)\n    \n#         boxes = outputs[i]['boxes'].data.cpu().numpy()\n#         scores = outputs[i]['scores'].data.cpu().numpy()\n        \n#         boxes = boxes[scores >= detection_threshold].astype(np.int32)\n#         scores = scores[scores >= detection_threshold]\n        \n        if image.shape[1] == 512:\n            boxes = (boxes*2).astype(np.int32).clip(min=0, max=1023)\n            \n        image_id = image_ids[i]\n        \n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n        \n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n\n        \n        results.append(result)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results[0:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = images[1].permute(1,2,0).cpu().numpy()\nboxes = outputs[0][1]['boxes']#.data.cpu().numpy()\nscores = outputs[0][1]['scores']#.data.cpu().numpy()\n\nboxes = boxes[scores >= detection_threshold].astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 2)\n    \nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(sample.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}