{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Here is my first public notebook.\n\n* Due to the limitation of YOLOv5, it is time to find a new model to have fun with. Although the current LB is not satisfiying, at least it works.\n* I am planning to improve performance of yolov4 by increasing training image size and data augmentation.\n* Thanks to dataset and TTA from [here](https://www.kaggle.com/shonenkov/oof-evaluation-mixup-efficientdet). \n* And the convert code is coming from [here](https://github.com/Tianxiaomo/pytorch-YOLOv4).","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!cp -r ../input/pytorchyolov4/tool .\n#!cp ../input/yolov4weights/yolo-wheat_best.weights .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\n\nfrom ensemble_boxes import *\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom ensemble_boxes import *\nimport torch\nimport random\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom torch.utils.data import Dataset,DataLoader\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport cv2\nimport gc\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom skopt import gp_minimize, forest_minimize\nfrom skopt.utils import use_named_args\nfrom skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret\nfrom skopt.space import Categorical, Integer, Real","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tool.utils import *\nfrom tool.torch_utils import *\nfrom tool.darknet2pytorch import Darknet\nimport argparse","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfgfile = '../input/yolov4weights/yolov4-obj.cfg'\nweightfile = '../input/yolov4weights/yolo-wheat_best.weights'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nm = Darknet(cfgfile)\n\nm.print_network()\nm.load_weights(weightfile)\n#print('Loading weights from %s... Done!' % (weightfile))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m.cuda()\nnum_classes = m.num_classes\nclass_names = load_class_names('../input/yolov4weights/wheat.names')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iou_threshold = 0.34\nskip_threshold = 0.31","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_valid_transforms():\n    return A.Compose([\n            A.Resize(height=704, width=704, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT_PATH = '../input/global-wheat-detection/test'\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg')\n        image = cv2.resize(image, (704, 704))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        #image /= 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = DatasetRetriever(\n    image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),\n    transforms=get_valid_transforms()\n)\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndata_loader = DataLoader(\n    dataset,\n    batch_size=2,\n    shuffle=False,\n    num_workers=1,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TTA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = 704\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTARotate180(BaseWheatTTA):\n    \n    def augment(self, image):\n        tmp = torch.rot90(image, 1, (1, 2))\n        return torch.rot90(tmp, 1, (1, 2))\n\n    def batch_augment(self, images):\n        tmp = torch.rot90(images, 1, (2, 3))\n        return torch.rot90(tmp, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        tmp = TTARotate90().deaugment_boxes(boxes)\n        return TTARotate90().deaugment_boxes(tmp)\n    \nclass TTARotate270(BaseWheatTTA):\n    \n    def augment(self, image):\n        tmp = TTARotate180().augment(image)\n        return torch.rot90(tmp, 1, (1, 2))\n\n    def batch_augment(self, images):\n        tmp = TTARotate180().batch_augment(images)\n        return torch.rot90(tmp, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        tmp = TTARotate180().deaugment_boxes(boxes)\n        return TTARotate90().deaugment_boxes(tmp)\n    \nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_det(index, det, score_threshold=0.25):\n    scores = det[index][:, 5].copy()\n    det = det[index][:, :4].copy()\n    bboxes = np.zeros((det.shape))\n    bboxes[:, 0] = ((det[:, 0] - det[:, 2] / 2) * 704).astype(int)\n    bboxes[:, 1] = ((det[:, 1] - det[:, 3] / 2) * 704).astype(int)\n    bboxes[:, 2] = ((det[:, 0] + det[:, 2] / 2) * 704).astype(int)\n    bboxes[:, 3] = ((det[:, 1] + det[:, 3] / 2) * 704).astype(int)\n    bboxes = (bboxes).clip(min = 0, max = 703).astype(int)\n    \n    indexes = np.where(scores>score_threshold)\n    bboxes = bboxes[indexes]\n    scores = scores[indexes]\n    return bboxes, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = TTACompose([\n    TTARotate270(),\n    #TTAVerticalFlip(),\n])\n\nfig, ax = plt.subplots(1, 3, figsize=(16, 6))\n\nimage, image_id = dataset[5]\n\nnumpy_image = image.permute(1,2,0).cpu().numpy().copy()\n\nax[0].imshow(numpy_image);\nax[0].set_title('original')\n\ntta_image = transform.augment(image)\nprint(tta_image.shape)\ntta_image_numpy = tta_image.permute(1,2,0).cpu().numpy().copy()\n\ndet = do_detect(m, tta_image.permute(1,2,0).cpu().numpy(), 0.001, 0.6)\ndet = np.array(det)\nprint(det.shape)\nboxes, scores = process_det(0, det)\n\n\nfor box in boxes:\n    cv2.rectangle(tta_image_numpy, (box[0], box[1]), (box[2],  box[3]), (0, 1, 0), 2)\n\nax[1].imshow(tta_image_numpy);\nax[1].set_title('tta')\n    \nboxes = transform.deaugment_boxes(boxes)\nfor box in boxes:\n    cv2.rectangle(numpy_image, (box[0], box[1]), (box[2],  box[3]), (0, 1, 0), 2)\n    \nax[2].imshow(numpy_image);\nax[2].set_title('deaugment predictions');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import product\n\ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None],\n                               [TTARotate180(), None],\n                               [TTARotate270(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_tta_predictions(images, score_threshold=0.25):\n    with torch.no_grad():\n        images = torch.stack(images).float().cuda()\n        predictions = []\n        #print('images.shape', images.shape)\n        for tta_transform in tta_transforms:\n            result = []\n            input_img = tta_transform.batch_augment(images.clone()).permute(0,2,3,1).cpu().numpy()\n            #print('input_img',input_img.shape)\n            det = do_detect(m, input_img, 0.4, 0.6)\n            #print([len(i) for i in det])\n            det = [np.array(i)for i in det]\n            #print(det)\n            for i in range(images.shape[0]):\n                boxes, scores = process_det(i, det)\n                boxes = tta_transform.deaugment_boxes(boxes.copy())\n                result.append({\n                    'boxes': boxes,\n                    'scores': scores,\n                })\n            predictions.append(result)\n    return predictions\n\nimport ensemble_boxes \ndef run_wbf(predictions, image_index, image_size=704, iou_thr=0.5, skip_box_thr=0.1, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in predictions]\n    boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfor j, (images, image_ids) in enumerate(data_loader):\n    break\n\npredictions = make_tta_predictions(images)\n\ni = 0\nsample = images[i].permute(1,2,0).cpu().numpy()\nboxes, scores, labels = run_wbf(predictions, image_index=i)\nboxes = boxes.round().astype(np.int32).clip(min=0, max=703)\n\nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (1, 0, 0), 1)\n\nax.set_axis_off()\nax.imshow(sample);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\n\nfor images, image_ids in data_loader:\n    try:\n        predictions = make_tta_predictions(images)\n    \n        for i, image in enumerate(images):\n            \n            boxes, scores, labels = run_wbf(predictions, image_index=i)\n            boxes = (boxes*(1024 / 704)).round().astype(np.int32).clip(min=0, max=1023)\n            image_id = image_ids[i]\n\n            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n            if len(boxes) > 0:\n                result = {\n                    'image_id': image_id,\n                    'PredictionString': format_prediction_string(boxes, scores)\n                }\n                results.append(result)\n    except:\n        for i, image in enumerate(images):\n            image_id = image_ids[i]\n            result = {\n                'image_id': image_id,\n                'PredictionString': ''\n            }\n            results.append(result)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}