{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!cp -r ../input/yolomodel/* .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from utils.datasets import *\nfrom utils.utils import *\nfrom models.experimental import *\nimport sys\nsys.path.insert(0, \"../input/weightedboxesfusion\")\nfrom ensemble_boxes import *\nimport torch\nimport glob\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = [\"../input/40efolds/best48fold0.pt\", \"../input/40efolds/best48fold1.pt\", \"../input/40efolds/best48fold2.pt\", \"../input/40efolds/best48fold3.pt\", \"../input/40efolds/best40fold4.pt\"]\nprint(len(weights))\nsource = \"../input/global-wheat-detection/test\"\nimgsz = 1024\nconf_t = 0.5\niou_t = 0.8\nbuilt_in_tta = True\n\ndevice = torch_utils.select_device(\"0\" if torch.cuda.is_available() else \"\")\nhalf = device.type != 'cpu'  # half precision only supported on CUDA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skip_box_thr = 0.43\niou_thr = 0.6\nscore_thr = 0.25\n\nws = [0.15, 0.15, 0.27, 0.15, 0.28]\n\ndef run_wbf(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.34, weights=None):\n    labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores))]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    return boxes, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def TTAImage(image, index):\n    image1 = image.copy()\n    if index==0: \n        rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image\n    elif index==1:\n        rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image2\n    elif index==2:\n        rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image3\n    elif index == 3:\n        return image1\n    \ndef rotBoxes90(boxes, im_w, im_h):\n    ret_boxes =[]\n    for box in boxes:\n        x1, y1, x2, y2 = box\n        x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2\n        x1, y1, x2, y2 = y1, -x1, y2, -x2\n        x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)\n        x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)\n        ret_boxes.append([x1a, y1a, x2a, y2a])\n    return np.array(ret_boxes)\n\ndef detect1Image(img, img0, model, device, aug):\n    img = img.transpose(2,0,1)\n\n    img = torch.from_numpy(img).to(device)\n    img = img.half() if half else img.float()  # uint8 to fp16/32\n\n    img /= 255.0\n    if img.ndimension() == 3:\n        img = img.unsqueeze(0)\n    \n    # Inference\n    pred = model(img, augment=aug)[0]\n    \n    # Apply NMS\n    pred = non_max_suppression(pred, conf_t, iou_t, merge=True, classes=None, agnostic=False)\n    \n    boxes = []\n    scores = []\n    for i, det in enumerate(pred):  # detections per image\n        # save_path = 'draw/' + image_id + '.jpg'\n        if det is not None and len(det):\n            # Rescale boxes from img_size to img0 size\n            det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape).round()\n\n            # Write results\n            for *xyxy, conf, cls in det:\n                boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])\n                scores.append(conf)\n\n    return np.array(boxes), np.array(scores) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def detect():\n    models = []\n\n    for w in weights if isinstance(weights, list) else [weights]:\n        models.append(torch.load(w, map_location=device)['model'].to(device).float().eval())\n        \n    dataset = LoadImages(source, img_size=imgsz)\n    img = torch.zeros((1, 3, imgsz, imgsz), device=device)  # init img\n    for model in models:\n        if half:\n            model.half()  # to FP16\n        _ = model(img.half() if half else img) if device.type != 'cpu' else None  # run once\n\n    all_paths, all_bboxes, all_confs = [], [], []\n    for p, img, img0, _ in dataset:\n        print()\n        img = img.transpose(1,2,0) # [H, W, 3]\n\n        all_paths.append(p)\n        m_box, m_score = [], []\n        for model in models:\n            enboxes = []\n            enscores = []\n            for i in range(4):\n                img1 = TTAImage(img, i)\n                boxes, scores = detect1Image(img1, img0, model, device, aug=False)\n                for _ in range(3-i):\n                    boxes = rotBoxes90(boxes, *img.shape[:2])            \n                enboxes.append(boxes)\n                enscores.append(scores)\n\n            boxes, scores = detect1Image(img, img0, model, device, aug=True)\n            enboxes.append(boxes)\n            enscores.append(scores) \n            \n            boxes, scores = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)    \n            boxes = boxes.astype(np.int32).clip(min=0, max=1023)\n            indices = scores >= score_thr\n            boxes = boxes[indices]\n            scores = scores[indices]\n            m_box.append(boxes)\n            m_score.append(scores)\n            \n        all_bboxes.append(m_box)\n        all_confs.append(m_score)\n    return all_paths, all_bboxes, all_confs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    res = detect()\npaths,all_boxes,all_confs = res\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resultsYOLO =dict()\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)\nbts = []\nsts = []\nfor row in range(len(paths)):\n    image_id = paths[row].split(\"/\")[-1].split(\".\")[0]\n    boxes, scores = run_wbf(all_boxes[row],all_confs[row], skip_box_thr = skip_box_thr, iou_thr = iou_thr, weights = None)\n    boxes = (boxes*1024/1024).astype(np.int32).clip(min=0, max=1023)\n    \n    resultsYOLO[image_id] = [boxes, scores]\n    \n    \n#     result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}\n#     results.append(result)\n#     bts.append(boxes)\n#     sts.append(scores)\n!rm -rf *\n# test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\n# test_df.to_csv('submission.csv', index=False)\n# test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resultsYOLO[\"2fd875eaa\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"/kaggle/input/weightedboxesfusion\")\n\nimport ensemble_boxes\n\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport os, re\nimport gc\nimport random\n\nimport torch\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor, FasterRCNN\nfrom torchvision.models.detection.backbone_utils import resnet_fpn_backbone\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/global-wheat-detection\"\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(DATA_DIR, \"sample_submission.csv\"))\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WheatDataset(Dataset):\n    def __init__(self, dataframe, image_dir, transforms=None):\n        super().__init__()\n\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n\n    def __len__(self) -> int:\n        return len(self.image_ids)\n\n    def __getitem__(self, idx: int):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n\n        records = self.df[self.df['image_id'] == image_id]\n    \n        if self.transforms:\n            sample = {\"image\": image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        return image, image_id\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fasterrcnn_resnet50_fpn(path,pretrained_backbone=False):\n\n    backbone = resnet_fpn_backbone('resnet50', pretrained_backbone)\n    model = FasterRCNN(backbone, 2)\n    model.load_state_dict(torch.load(path))\n    model.to(DEVICE)\n    model.eval()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_101( path,pretrained=False):\n    backbone = resnet_fpn_backbone('resnet101', pretrained=pretrained)\n    model = FasterRCNN(backbone, num_classes=2)\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 2)\n    model.load_state_dict(torch.load(path))\n    model.to(DEVICE)\n    model.eval()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model =get_model_101(\"../input/faster-cnn-final-model/fastercnn_resnet_101_145.pth\")\nmodel2 =get_model_101(\"../input/faster-cnn-final-model/fastercnn_resnet_101_145.pth\")\nmodel3=get_model_101(\"../input/faster-cnn-101-155-fix/fastercnn_resnet_101_fix_155.pth\")\nmodel4= get_model_101(\"../input/fastercnn-155/fastercnn_101_155.pth\")\nmodel5= get_model_101(\"../input/faster-cnn-125/fastercnn_resnet_101_125.pth\")\nfaster_CNN_50_90=fasterrcnn_resnet50_fpn(\"../input/isfix-model/fastercnn_50_fix.pth\")\nfaster_CNN_50_90_2=fasterrcnn_resnet50_fpn(\"../input/isfix-model/fastercnn_50_fix.pth\")\nfaster_CNN_50_105=fasterrcnn_resnet50_fpn(\"../input/faster-50-fix-115/fastercnn_50_fix_115.pth\")\n\nmodels=[model,model3,model4,model5,faster_CNN_50_105,faster_CNN_50_90,faster_CNN_50_90_2\n       ,model2\n       ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ensemble_boxes import *\n\ndef make_ensemble_predictions(images):\n    images = list(image.to(DEVICE) for image in images)    \n    result = []\n    for model in models:\n        with torch.no_grad():\n            outputs = model(images)\n            result.append(outputs)\n            del model\n            gc.collect()\n            torch.cuda.empty_cache()\n    return result\n\ndef run_wbf_ensemble(predictions, image_index, image_size=1024, iou_thr=0.45, skip_box_thr=0.43, weights=None):\n    boxes = [prediction[image_index]['boxes'].data.cpu().numpy()/(image_size-1) for prediction in predictions]\n    scores = [prediction[image_index]['scores'].data.cpu().numpy() for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]) for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_transforms():\n    return A.Compose([\n            ToTensorV2(p=1.0)\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntest_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, \"test\"), get_test_transforms())\n\ntest_data_loader = DataLoader(\n    test_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=1,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = 1024\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]] \n        res_boxes[:, [1,3]] = boxes[:, [0,2]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import product\n\ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_tta_predictions(images, score_threshold=0.57):\n    with torch.no_grad():\n        images = torch.stack(images).float().to(DEVICE)\n        predictions = []\n        for tta_transform in tta_transforms:\n            result = []\n            #ensemble predict\n            outputs = make_ensemble_predictions(tta_transform.batch_augment(images.clone()))\n            #outputs = model(tta_transform.batch_augment(images.clone()))\n\n            for i, image in enumerate(images):\n                #chose the boxes and scores\n                boxes, scores, labels = run_wbf_ensemble(outputs, image_index=i)\n                #boxes = outputs[i]['boxes'].data.cpu().numpy()   \n                #scores = outputs[i]['scores'].data.cpu().numpy()\n                \n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes = tta_transform.deaugment_boxes(boxes.copy())\n                result.append({\n                    'boxes': boxes,\n                    'scores': scores[indexes],\n                })\n            predictions.append(result)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.4, skip_box_thr=0.43, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in predictions]\n    boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resultsFASTER = {}\n\nfor images, image_ids in test_data_loader:\n\n    predictions = make_tta_predictions(images)\n    for i, image in enumerate(images):\n        boxes, scores, labels = run_wbf(predictions, image_index=i)\n        boxes = boxes.round().astype(np.int32).clip(min=0, max=1023)\n        image_id = image_ids[i]\n        \n        resultsFASTER[image_id] = [boxes, scores]\n        \n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_wbf(boxes,scores, image_size=1024, iou_thr=0.6, skip_box_thr=0.43, weights=None):\n    labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores))]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    return boxes, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_string = []\nbts, sts, ids = [], [], []\nresults = dict()\nfor image_id, faster in resultsFASTER.items():\n    yolo = resultsYOLO[image_id]\n    scores_ = [faster[1], yolo[1],yolo[1]]\n    boxes_ = [faster[0], yolo[0],yolo[0]]\n    boxes, scores = run_wbf(boxes_, scores_, iou_thr = 0.45, skip_box_thr=0.01)\n    boxes = boxes.round().astype(np.int32).clip(min=0, max=1023)\n    \n    indices = scores >= 0.3\n    boxes = boxes[indices]\n    scores = scores[indices]\n    \n    boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n    \n    result = {'image_id': image_id, 'PredictionString': format_prediction_string(boxes, scores)}\n    pred_string.append(result)\n    ids.append(image_id)\n    bts.append(boxes)\n    sts.append(scores)\n\n    \ntest_df = pd.DataFrame(pred_string, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"../input/global-wheat-detection/test/\"\nfont = cv2.FONT_HERSHEY_SIMPLEX \nfig = plt.figure(figsize=[30,30])\nbts =[b.round().astype(np.int32).clip(min=0, max=1023) for b in bts]\nfor i in range(len(image_id)):\n    fig.add_subplot(4, 3, i+1)\n\n    p = PATH + ids[i] + \".jpg\"\n    image = cv2.imread(p, cv2.IMREAD_COLOR)\n    color = (255, 0, 0) \n\n    for b, s in zip(bts[i], sts[i]):\n        if s> 0.2:\n            b2 = int(b[0]+ b[2])\n            b3 = int(b[1] + b[3])\n            b[0] = int(b[0])\n            b[1] = int(b[1])\n\n            image = cv2.rectangle(image, (b[0],b[1]), (b2,b3), (255,0,0), 2) \n            image = cv2.putText(image, '{:.2}'.format(s), (b[0]+np.random.randint(3),b[1]), font, 1, color, 2, cv2.LINE_AA)\n    plt.title(ids[i])\n    plt.imshow(image)\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}