{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Global Wheat Detection - Liran Nachman\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"![](https://d14peyhpiu05bf.cloudfront.net/uploads/2019/01/shutterstock_488899324Wheat-Grains-770x350.jpg)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Hey everybody, my name is Liran Nachman,I am software engineer student.\nthis is my first competition in kaggle, so wish me luck :) \n\nIn this kernel I based on couple things:\n\n -  RCNN with backbone resnet50\n -  Image transformations \n -  Weighted Boxes Fusion instead Non Max Suppression \n -  Test Time Augmentation \n \nJust want thank you :\n - https://github.com/ZFTurbo/Weighted-Boxes-Fusion\n - https://www.kaggle.com/guizengyou/tta-more-transforms ( thank you gzYou)\n - https://www.kaggle.com/pestipeti/competition-metric-details-script ( helper functions)\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Import libraries**","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/weightedboxesfusion\")\nimport glob\nimport os\nimport gc\nimport random\nfrom ensemble_boxes import *\nimport torch\nfrom  torch.utils.data import *\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom torchvision.transforms import transforms\nimport albumentations as A\nfrom albumentations import (\n    BboxParams,\n    HorizontalFlip,\n    VerticalFlip,\n    Resize,\n    CenterCrop,\n    RandomCrop,\n    Crop,\n    Compose\n)\nfrom tqdm import tqdm\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nimport torchvision\nimport cv2\nfrom matplotlib import pyplot as plt\nimport time\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom itertools import product\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Helper functions**","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def visualize(**images):\n    \"\"\"PLot images in one row.\"\"\"\n    n = len(images)\n    plt.figure(figsize=(16, 5))\n    for i, (name, image) in enumerate(images.items()):\n        plt.subplot(1, n, i + 1)\n        plt.xticks([])\n        plt.yticks([])\n        plt.title(' '.join(name.split('_')).title())\n        plt.imshow(image)\n    plt.show()\n\n\nclass Metrics:\n    @staticmethod\n    def calculate_iou(gt, pr, form='pascal_voc') -> float:\n        \"\"\"Calculates the Intersection over Union.\n\n        Args:\n            gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n            pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n            form: (str) gt/pred coordinates format\n                - pascal_voc: [xmin, ymin, xmax, ymax]\n                - coco: [xmin, ymin, w, h]\n        Returns:\n            (float) Intersection over union (0.0 <= iou <= 1.0)\n        \"\"\"\n        if form == 'coco':\n            gt = gt.copy()\n            pr = pr.copy()\n\n            gt[2] = gt[0] + gt[2]\n            gt[3] = gt[1] + gt[3]\n            pr[2] = pr[0] + pr[2]\n            pr[3] = pr[1] + pr[3]\n\n        # Calculate overlap area\n        dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n\n        if dx < 0:\n            return 0.09\n        dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n        if dy < 0:\n            return 0.0\n\n        overlap_area = dx * dy\n\n        # Calculate union area\n        union_area = (\n                (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n                (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n                overlap_area\n        )\n\n        return overlap_area / union_area\n\n    @staticmethod\n    def find_best_match(gts, pred, pred_idx, threshold=0.5, form='pascal_voc', ious=None) -> int:\n        \"\"\"Returns the index of the 'best match' between the\n        ground-truth boxes and the prediction. The 'best match'\n        is the highest IoU. (0.0 IoUs are ignored).\n\n        Args:\n            gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n            pred: (List[Union[int, float]]) Coordinates of the predicted box\n            pred_idx: (int) Index of the current predicted box\n            threshold: (float) Threshold\n            form: (str) Format of the coordinates\n            ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n        Return:\n            (int) Index of the best match GT box (-1 if no match above threshold)\n        \"\"\"\n        best_match_iou = -np.inf\n        best_match_idx = -1\n        for gt_idx in range(len(gts)):\n\n            if gts[gt_idx][0] < 0:\n                # Already matched GT-box\n                continue\n\n            iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n            if iou < 0:\n                iou = Metrics.calculate_iou(gts[gt_idx], pred, form=form)\n\n                if ious is not None:\n                    ious[gt_idx][pred_idx] = iou\n\n            if iou < threshold:\n                continue\n\n            if iou > best_match_iou:\n                best_match_iou = iou\n                best_match_idx = gt_idx\n\n        return best_match_idx\n\n    @staticmethod\n    def calculate_precision(gts, preds, threshold=0.5, form='coco', ious=None) -> float:\n        \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n        Args:\n            gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n            preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n                   sorted by confidence value (descending)\n            threshold: (float) Threshold\n            form: (str) Format of the coordinates\n            ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n        Return:\n            (float) Precision\n        \"\"\"\n        n = len(preds)\n        tp = 0\n        fp = 0\n\n        for pred_idx in range(n):\n\n            best_match_gt_idx = Metrics.find_best_match(gts, preds[pred_idx], pred_idx,\n                                                        threshold=threshold, form=form, ious=ious)\n\n            if best_match_gt_idx >= 0:\n                # True positive: The predicted box matches a gt box with an IoU above the threshold.\n                tp += 1\n                # Remove the matched GT box\n                gts[best_match_gt_idx] = -1\n            else:\n                # No match\n                # False positive: indicates a predicted box had no associated gt box.\n                fp += 1\n\n        # False negative: indicates a gt box had no associated predicted box.\n        fn = (gts.sum(axis=1) > 0).sum()\n\n        return tp / (tp + fp + fn)\n\n    @staticmethod\n    def calculate_image_precision(gts, preds, thresholds=(0.5,), form='coco') -> float:\n        \"\"\"Calculates image precision.\n\n        Args:\n            gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n            preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n                   sorted by confidence value (descending)\n            thresholds: (float) Different thresholds\n            form: (str) Format of the coordinates\n\n        Return:\n            (float) Precision\n        \"\"\"\n        n_threshold = len(thresholds)\n        image_precision = 0.0\n\n        ious = np.ones((len(gts), len(preds))) * -1\n        # ious = None\n\n        for threshold in thresholds:\n            precision_at_threshold = Metrics.calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                                 form=form, ious=ious)\n            image_precision += precision_at_threshold / n_threshold\n\n        return image_precision\n    \n    \nclass Averager:\n    def __init__(self):\n        self.current_total = 0.0\n        self.iterations = 0.0\n\n    def send(self, value):\n        self.current_total += value\n        self.iterations += 1\n\n    @property\n    def value(self):\n        if self.iterations == 0:\n            return 0\n        else:\n            return 1.0 * self.current_total / self.iterations\n\n    def reset(self):\n        self.current_total = 0.0\n        self.iterations = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = 512\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"# Functions to visualize bounding boxes and class labels on an image. \n# Based on https://github.com/facebookresearch/Detectron/blob/master/detectron/utils/vis.py\n\nBOX_COLOR = (255, 0, 0)\nTEXT_COLOR = (255, 255, 255)\n\n\ndef visualize_bbox(img, bbox, class_id, class_idx_to_name, color=BOX_COLOR, thickness=2):\n    x_min, y_min, x_max, y_max = bbox\n    x_min, y_min, x_max, y_max =  int(x_min), int(y_min), int(x_max), int(y_max)\n    cv2.rectangle(img, (x_min, y_min), (x_max, y_max), color=color, thickness=thickness)\n    class_name = class_idx_to_name[class_id]\n    ((text_width, text_height), _) = cv2.getTextSize(class_name, cv2.FONT_HERSHEY_SIMPLEX, 0.35, 1)    \n    cv2.rectangle(img, (x_min, y_min - int(1.3 * text_height)), (x_min + text_width, y_min), BOX_COLOR, -1)\n    cv2.putText(img, class_name, (x_min, y_min - int(0.3 * text_height)), cv2.FONT_HERSHEY_SIMPLEX, 0.35,TEXT_COLOR, lineType=cv2.LINE_AA)\n    return img\n\n\ndef visualizeTarget(image,target, category_id_to_name={1 : \"Wheat\"}):\n    img = image.copy()\n    for idx, bbox in enumerate(target['boxes']):\n        img = visualize_bbox(img, bbox, target['labels'][idx], category_id_to_name)\n#     plt.figure(figsize=(12, 12))\n#     plt.imshow(img)\n    return img\n    \ndef get_aug(aug, min_area=0., min_visibility=0.):\n    return Compose(aug, bbox_params=BboxParams(format='pascal_voc', min_area=min_area, \n                                               min_visibility=min_visibility, label_fields=['labels']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Dataloader & train function & WBF**","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":false},"cell_type":"code","source":"class WheatDataset(Dataset):\n    def __init__(self, dataframe, image_dir, transforms=None):\n        super().__init__()\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)  # reading an image\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)  # changing color space BGR --> RGB\n        image /= 255.0\n\n        boxes = records[['x', 'y', 'w', 'h']].to_numpy()\n        area = (boxes[:, 3]) * (boxes[:, 2])  # Calculating area of boxes W * H\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]  # upper coordinate X + W\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]  # lower coordinate Y + H\n        area = torch.as_tensor(area, dtype=torch.float32)\n\n        # there is only one class\n        labels = torch.ones((records.shape[0],), dtype=torch.int64)\n\n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((records.shape[0],), dtype=torch.int64)\n\n        target = {'bboxes': boxes, 'labels': labels, 'image_id': torch.tensor([index]), 'area': area, 'iscrowd': iscrowd}\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': target['bboxes'],\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            \n            image = sample[\"image\"]\n            \n            target['bboxes'] = torch.as_tensor(sample['bboxes'], dtype=torch.float32)\n            target['bboxes'] =  target['bboxes'].reshape(-1, 4)\n            target[\"boxes\"] = target[\"bboxes\"]\n            \n            del target['bboxes']\n            return image,target, f'{self.image_dir}/{image_id}.jpg'  # image Tensor , target with boxes , path to image\n\n    def __len__(self):\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"class DatasetUtils:\n    @staticmethod\n    def collate_fn(batch):\n        return tuple(zip(*batch))\n\n    @staticmethod\n    def preprocessing_csv(data_frame):\n        # seperate bbox to x,y,w,h columns\n        seperator = lambda x: np.fromstring(x[1:-1], sep=',')\n        bbox = np.stack(data_frame['bbox'].apply(seperator))\n        for i, dim in enumerate(['x', 'y', 'w', 'h']):\n            data_frame[dim] = bbox[:, i]\n        data_frame.drop(columns='bbox', inplace=True)\n\n    @staticmethod\n    def splitData(all_data_records: pd.DataFrame, test_size=0.33):\n        image_ids = all_data_records['image_id'].unique()\n        train_ids, valid_ids = train_test_split(image_ids, test_size=test_size, random_state=42)\n        valid_df = all_data_records[all_data_records['image_id'].isin(valid_ids)]\n        train_df = all_data_records[all_data_records['image_id'].isin(train_ids)]\n        return train_df, valid_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"def run_wbf(predictions,weights=None,image_size=512,iou_thr=0.5,skip_box_thr=0.43):\n    boxes_list = [(pred[\"boxes\"] / (image_size-1)).tolist() for pred in predictions]\n    scores_list = [pred[\"scores\"].tolist() for pred in predictions]\n    labels_list = [np.ones(len(score)).astype(int).tolist() for score in scores_list]\n    boxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n\n    return boxes, scores, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class TrainUtils:\n    @staticmethod\n    def trainModels(models, train_dataloader, valid_dataloader, device, num_epochs=1, valid_pred_min=0.65):\n        for model in models:\n            print(\"Starting train model \", str(model.__name__))\n            model.to(device)\n            # construct an optimizer\n            params = [p for p in model.parameters() if p.requires_grad]\n            optimizer = torch.optim.SGD(params, lr=0.005,\n                                        momentum=0.9, weight_decay=0.0005)\n\n            train_hist = Averager()\n            t = 0\n            for epoch in range(num_epochs):\n                model.train()\n                train_hist.reset()\n                for images, targets,image_path in train_dataloader:\n                    model.train()\n                    images = list(image.to(device) for image in images)\n                    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n                    loss_dict = model(images, targets)\n                    losses = sum(loss for loss in loss_dict.values())\n                    train_loss = losses.item()\n                    train_hist.send(train_loss)\n                    optimizer.zero_grad()\n                    losses.backward()\n                    optimizer.step()\n                    t += 1\n\n                model.eval()\n                validation_image_precisions = []\n                iou_thresholds = [x for x in np.arange(0.5, 0.76, 0.05)]\n                for images, targets,image_path in valid_dataloader:\n                    images = list(image.to(device) for image in images)\n                    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n                    with torch.no_grad():\n                        outputs = model(images)\n\n                    for i, image in enumerate(images):\n                        boxes = outputs[i]['boxes'].data.cpu().numpy()\n                        scores = outputs[i]['scores'].data.cpu().numpy()\n                        gt_boxes = targets[i]['boxes'].cpu().numpy()\n                        preds_sorted_idx = np.argsort(scores)[::-1]\n                        preds_sorted = boxes[preds_sorted_idx]\n                        image_precision = Metrics.calculate_image_precision(preds_sorted,\n                                                                            gt_boxes,\n                                                                            thresholds=iou_thresholds,\n                                                                            form='coco')\n                        validation_image_precisions.append(image_precision)\n\n                valid_prec = np.mean(validation_image_precisions)\n                print(\"Validation Precision: {0:.4f}\".format(valid_prec))\n\n                # print training/validation statistics\n                print('Epoch: {} \\tTraining Loss: {:.6f}'.format(\n                    epoch,\n                    train_loss\n                ))\n\n                if valid_prec >= valid_pred_min:\n                    print('Validation precision increased({:.6f} --> {:.6f}).  Saving model ...'.format(\n                        valid_pred_min,\n                        valid_prec))\n                    torch.save(model.state_dict(), 'faster_rrcnn_' + str(model.__name__) + '.pth')\n                    valid_pred_min = valid_prec\n            torch.save(model.state_dict(), 'faster_rrcnn_' + str(model.__name__) + '_' + str(time.time()) + '.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class FpnResenet50:\n    @staticmethod\n    def getNet():\n        fpn_resnet = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False,pretrained_backbone=False)\n        num_classes = 2  # 1 class (wheat) + background\n        # get number of input features for the classifier\n        in_features = fpn_resnet.roi_heads.box_predictor.cls_score.in_features\n        # replace the pre-trained head with a new one\n        fpn_resnet.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n        fpn_resnet.__name__ = \"fpn_resnet\"\n        return fpn_resnet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing on data before train","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"DATA_ROOT_PATH = '../input/global-wheat-detection'\nall_wheat_dataset_train = pd.read_csv(DATA_ROOT_PATH+'/train.csv')\nDatasetUtils.preprocessing_csv(all_wheat_dataset_train)\nall_wheat_dataset_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explorer the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## source distributions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"images_id_without_duplication = all_wheat_dataset_train.drop_duplicates(\"image_id\")\nimages_id_without_duplication.groupby(\"source\").count().plot(kind=\"bar\",figsize=(12,6),title=\"Number of images by sources\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## bboxs distributions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df = all_wheat_dataset_train.image_id.value_counts().to_frame()\ncounts = df.image_id.values\nimages_id = df.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,6))\nax.bar(images_id,counts)\nax.set_ylabel('Number of bboxs')\nax.set_title('Number of bboxs by mages')\nax.set_xlabel('images')\nax.set_xticks(\"\")\nax.set_xticklabels(\"labels\")\nax.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### please find the image with the most bboxs ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"max_bbox =  all_wheat_dataset_train.groupby(\"image_id\").agg(['count']).max()[0]\nimage_id_max =  all_wheat_dataset_train.groupby(\"image_id\").agg(['count']).idxmax()[0]\nprint(\"Image {} have Max bbox : {}\".format(image_id_max,max_bbox))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = image_id_max\n    \nrecords = all_wheat_dataset_train[all_wheat_dataset_train['image_id'] == image_id]\n\nprint(len(records))\nimage = cv2.imread(f'{DATA_ROOT_PATH}/train/{image_id}.jpg', cv2.IMREAD_COLOR)  # reading an image\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)  # changing color space BGR --> RGB\nimage /= 255.0\nboxes = records[['x', 'y', 'w', 'h']].to_numpy()\narea = (boxes[:, 3]) * (boxes[:, 2])  # Calculating area of boxes W * H\nboxes[:, 2] = boxes[:, 0] + boxes[:, 2]  # upper coordinate X + W\nboxes[:, 3] = boxes[:, 1] + boxes[:, 3]  # lower coordinate Y + H\ntarget = { \"boxes\" : boxes , \"labels\" : np.ones(len(boxes))}\nimg = visualizeTarget(image,target) \nvisualize(Image = img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split data to train and validation","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_df,valid_df = DatasetUtils.splitData(all_data_records=all_wheat_dataset_train)\ntrain_df.shape,valid_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image transforms\n\nI tried a lot of transforms,after research I found that when adding random crop getting better results , was 0.4971 and **after** **0.5241**.\nAlso VerticalFlip, Blur, Gray scale boosting the results.\n","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"aug_trans = get_aug([HorizontalFlip(p=0.5),\n                     VerticalFlip(p=0.5),\n                     A.ToGray(p=0.3),\n                     A.GaussianBlur(p=0.3),\n                     A.RandomBrightnessContrast(p=0.7),\n                     RandomCrop(p=0.5,height=512,width=512),\n                     Resize(width=512,height=512),\n                     ToTensorV2(p=1)])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset and data loader","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_dataset = WheatDataset(train_df,DATA_ROOT_PATH+\"/train\",transforms=aug_trans)\nvalid_dataset = WheatDataset(valid_df,DATA_ROOT_PATH+\"/train\",transforms=aug_trans)\n\ntrain_dataloader = DataLoader(train_dataset,batch_size=8,collate_fn=DatasetUtils.collate_fn,num_workers=8)\nvalid_dataloader = DataLoader(valid_dataset,batch_size=2,collate_fn=DatasetUtils.collate_fn,num_workers=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visual our dataset ( please noted to the random crop that created a zoom that improve results) ","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"for data in valid_dataloader:\n    images, targets, image_path = data\n    targets = [{k: v.to(device).detach().cpu().numpy() for k, v in t.items()} for t in targets]\n    images = list(image.to(device) for image in images)\n    with torch.no_grad():\n        for image, target in zip(images, targets):\n            image = image.squeeze(0).permute(1,2,0)\n            image = image.detach().cpu().numpy()\n            img = visualizeTarget(image,target) \n            visualize(image =img)\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RCNN with backbone resnet50","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"resnet50 = FpnResenet50.getNet()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"#%time TrainUtils.trainModels([resnet50], train_dataloader, valid_dataloader, device, num_epochs=25, valid_pred_min=0.55)\n#torch.save(resnet50.state_dict(), 'resnet_taa_finish_25_epoch_' + str(resnet50.__name__) + '_' + str(time.time()) + '.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Testing with TTA over WBF ","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"resnet50.load_state_dict(torch.load('../input/resnet50-tta/resnet_taa_finish_20_full_data_v1_fpn_resnet_1593370667.8672676.pth'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Time Augmentation \n\nWe boosted our results with TTA transforms.\nEach image in the test dataset, we create an n (for example 5) mutation images ( not the same image).\nfollowing those steps :\n\n- each mutation image :\n    - predict image and getting boxes and scores\n    - deaugment boxes to original coordination of original image\n    - append boxes and scores to a list of boxes\n- doing WBF on all list of boxes and list of scores that give us final boxes and scores .\n\n**Why this is work?**\n\nWhen we predict one time, we got some error in our model. \nbut if we try a couple of times the same image but with little mutation,\nthen average the results we also average the error and our accuracy improved. \n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## We are creating an array of TTA compose\n\nwe selected a random compose to generated differences images ","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"tta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TTA demonstration","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def tta_demonstration(path):\n    iou_thr = 0.4\n    detection_threshold = 0.5\n    resnet50.cuda()\n    resnet50.eval()\n\n    image_test_numpy = plt.imread(path).astype(np.float32)\n    image_test_numpy /= 255.0\n    \n    t = Compose([Resize(width=512,height=512),ToTensorV2()])\n    data = { \"image\": image_test_numpy}\n    data = t(**data)\n    \n    \n    image_test_tensor = data[\"image\"]\n    image_test_tensor = image_test_tensor.squeeze(0)\n    selected_tta = random.choice(tta_transforms)\n    \n    \n    tta_image = selected_tta.augment(image_test_tensor) ## need to be random\n    outputs = resnet50(tta_image.unsqueeze(0).cuda())\n    boxes = outputs[0]['boxes'].data.detach().cpu().numpy()\n    scores = outputs[0]['scores'].data.detach().cpu().numpy() \n    \n    boxes = boxes[scores >= detection_threshold]\n    scores = scores[scores >= detection_threshold]\n    original_boxes  = selected_tta.deaugment_boxes(boxes.copy())\n    \n    tta_image_numpy = tta_image.permute(1,2,0).detach().cpu().numpy()\n    image_test_numpy = image_test_tensor.permute(1,2,0).detach().cpu().numpy()\n\n    return image_test_numpy,tta_image_numpy,boxes,original_boxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"test_images_paths = glob.glob(os.path.join('/kaggle/input/global-wheat-detection/test/*'))\n\nfor image_path in test_images_paths:\n    image_test_numpy ,image_tta_numpy,boxes_tta,original_boxes = tta_demonstration(image_path)\n    original_with_boxes = visualizeTarget(image_test_numpy\n                                  ,{\"boxes\": original_boxes,'labels' : np.ones(len(original_boxes))})\n\n    tta_with_boxes = visualizeTarget(image_tta_numpy\n                                  ,{\"boxes\": boxes_tta,'labels' : np.ones(len(boxes_tta))})\n\n    visualize(\n\n        original = image_test_numpy,\n        image_with_tta_boxes = tta_with_boxes,\n        back_original_with_bbox = original_with_boxes\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"resnet50.cuda()\nresnet50.eval()\n\niou_thr = 0.4\ndetection_threshold = 0.5\nn = 20\n\ntest_images_paths = glob.glob(os.path.join('/kaggle/input/global-wheat-detection/test/*'))\nsubmission = []\n\nfor image_path in tqdm(test_images_paths):\n    image = cv2.imread(image_path, cv2.IMREAD_COLOR)  # reading an image\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)  # changing color space BGR --> RGB\n    image /= 255.0\n    t = Compose([Resize(width=512,height=512),ToTensorV2()])\n    data = { \"image\": image}\n    data = t(**data)\n    image = data[\"image\"]\n    image = image.squeeze(0)\n   \n\n    predictions = []\n    \n    for i in range(n):\n        ## create tta image\n        selected_tta = random.choice(tta_transforms)\n        tta_image = selected_tta.augment(image) ## need to be random\n        outputs = resnet50(tta_image.unsqueeze(0).cuda())\n        boxes = outputs[0]['boxes'].data.detach().cpu().numpy()\n        scores = outputs[0]['scores'].data.detach().cpu().numpy()\n        boxes = boxes[scores >= detection_threshold]\n        scores = scores[scores >= detection_threshold]\n        original_boxes  = selected_tta.deaugment_boxes(boxes)\n        predictions.append({\"boxes\"  : original_boxes,'scores': scores})\n    \n    boxes, scores, labels = run_wbf(predictions,iou_thr=iou_thr,image_size=512)\n    \n    boxes = boxes * 1024\n        \n    \n    image = image.permute(1,2,0).detach().cpu().numpy()\n    \n    image = cv2.resize(image,(1024,1024))\n    original_with_boxes = visualizeTarget(image\n                              ,{\"boxes\": boxes,'labels' : np.ones(len(boxes))})\n    \n    visualize(original=original_with_boxes)\n\n    prediction_string = []\n    \n    for (boxes, s) in zip(boxes,scores):\n        x_min , y_min , x_max,y_max = boxes\n        x = round(x_min)\n        y = round(y_min)\n        h = round(x_max-x_min)\n        w = round(y_max-y_min)\n        prediction_string.append(f\"{s} {x} {y} {h} {w}\")\n    prediction_string = \" \".join(prediction_string)\n    \n    image_name = image_path.split(\"/\")[-1].split(\".\")[0]\n    submission.append([image_name, prediction_string])\n \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"SUBMISSION_PATH = '/kaggle/working'\nsubmission_id = 'submission'\ncur_submission_path = os.path.join(SUBMISSION_PATH, '{}.csv'.format(submission_id))\nsample_submission = pd.DataFrame(submission, columns=[\"image_id\",\"PredictionString\"])\nsample_submission.to_csv(cur_submission_path, index=False)\n\n\n# validate creating csv \nsubmission_df = pd.read_csv(cur_submission_path)\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you for reading my kernel,If this helps somebody please vote :)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}