{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import useful tools\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport numba \nimport ast\nimport os\nimport re\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom glob import glob\nfrom PIL import Image\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor, FasterRCNN\nfrom torchvision.models.detection.backbone_utils import resnet_fpn_backbone\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom collections import namedtuple\n\n# import data visualization\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\n\nfrom bokeh.plotting import figure\nfrom bokeh.io import output_notebook, show, output_file\nfrom bokeh.models import ColumnDataSource, HoverTool, Panel\nfrom bokeh.models.widgets import Tabs\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\n# import data augmentation\nimport albumentations as albu\nfrom albumentations.pytorch.transforms import ToTensorV2\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"base_path = \"/kaggle/input/global-wheat-detection/\"\ntrain_dir = f'{base_path}/train/'\ntest_dir = f'{base_path}/test/'\ntrain_img = glob(train_dir + '*') # create a list of train images\ntest_img = glob(test_dir + '*') # create a list of test images\n#print('Number of train images is {}'.format(len(train_img))) # 3422 train images\n#print('Number of test images is {}'.format(len(test_img))) # 10 test images\ntrain_csv_path = f'{base_path}/train.csv'\ntrain = pd.read_csv(train_csv_path) # load dataframe with bboxes\n#print(train['bbox'].head(4)) # Show four first rows of the col. bbox\nweights_dir = '/kaggle/input/global-wheat-detection-public' \nweights_file = f'{weights_dir}/fasterrcnn_resnet50_fpn_best.pt'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a dataframe with all train images\ntrain_images = pd.DataFrame([img.split('/')[-1][:-4] for img in train_img])\ntrain_images.columns=['image_id']\n\n# Merge all train images with the bounding boxes dataframe\ntrain_images = train_images.merge(train, on='image_id', how='left')\n\n# replace nan values with zeros\ntrain_images['bbox'] = train_images.bbox.fillna('[0,0,0,0]')\ntrain_images.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images.drop(columns=['width'], inplace=True) # delete width column\ntrain_images.drop(columns=['height'], inplace=True) # delete height column\n\n# split bbox column\nbbox_items = train_images.bbox.str.split(',', expand=True)\ntrain_images['x_min'] = bbox_items[0].str.strip('[ ').astype(float)\ntrain_images['y_min'] = bbox_items[1].str.strip(' ').astype(float)\ntrain_images['width'] = bbox_items[2].str.strip(' ').astype(float)\ntrain_images['height'] = bbox_items[3].str.strip(' ]').astype(float)\ntrain_images.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# adding 2 col. x_max and y_max\ntrain_images['x_max'] = train_images.apply(lambda x: x.x_min + x.width, axis=1)\ntrain_images['y_max'] = train_images.apply(lambda y: y.y_min + y.height, axis=1)\n# train_images['x_max'] = train_images.apply(lambda x: x.x_min + x.bbox_width, axis=1)\n# train_images['y_max'] = train_images.apply(lambda y: y.y_min + y.bbox_height, axis=1)\n# train_img.drop(columns=['bbox'], inplace=True) # Delete bbox column\n\n#adding column for bbox_area\ntrain_images['bbox_area'] = train_images.apply(lambda z: z.width * z.height, axis=1)\n\ntrain_images.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print('{} images without wheat heads.'.format(len(train_images) - len(train))) #49 images without wheat heads","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking bounding box coordinates\n# print(max(train_images['x_max'])) # 1024\n# print(max(train_images['y_max'])) #1024\n# print(min(train_images['x_min'])) #0\n# print(min(train_images['y_min'])) #0\nx_max = np.array(train_images['x_max'].values.tolist()) # Changing x_max from dataframe to a list\ny_max = np.array(train_images['y_max'].values.tolist()) # Changing y_max from dataframe to a list\ntrain_images['x_max'] = np.where(x_max > 1024, 1024, x_max).tolist()\ntrain_images['y_max'] = np.where(y_max > 1024, 1024, y_max).tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"image examples\"\"\"\ndef find_bboxes(df, image_id):\n    img_bbox = df[df['image_id'] == image_id]\n    bboxes = []\n    for col, row in img_bbox.iterrows():\n        bboxes.append((row.x_min, row.y_min, row.width, row.height))\n#         bboxes.append((row.x_min, row.y_min, row.bbox_width, row.bbox_height))\n    return bboxes\ndef plt_img(df, rows=3, column=3, title='Image examples'):\n    fig, axs = plt.subplots(rows, column, figsize=(30, 30))\n    for row in range(rows):\n        for col in range(column):\n            idx = np.random.randint(len(df), size=1)[0]\n            img_num = df.iloc[idx].image_id\n            img = Image.open(train_dir + img_num + '.jpg')\n            axs[row, col].imshow(img)\n            bboxes = find_bboxes(df, img_num)\n\n            for bbox in bboxes:\n                rect_patch = patches.Rectangle((bbox[0], bbox[1]), bbox[2], bbox[3], linewidth=2, edgecolor='y', facecolor='none')\n                axs[row, col].add_patch(rect_patch)\n            axs[row, col].axis('off')\n    plt.suptitle(title)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt_img(train_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"count the number of bboxes per image\"\"\"\ntrain_images['num_bboxes'] = train_images.apply(lambda x: 1 if np.isfinite(x.width) else 0, axis=1)\ntrain_img_count = train_images.groupby('image_id').sum().reset_index() # count the num of bboxes in each image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def hist_hover(df, column, colors=[\"#94c8d8\", \"#ea5e51\"], bins=30, title=''):\n    # build histogram data with np\n    hist, edges = np.histogram(df[column], bins=bins)\n    hist_df = pd.DataFrame({column: hist,\n                            \"left\": edges[:-1],\n                            \"right\": edges[1:]})\n    hist_df['interval'] = [\"%d to %d\" % (left, right) for left, right in zip(hist_df['left'], hist_df['right'])]\n    # %d is a numeric or decimalplaceholder\n    # create col. data source in bokeh\n    src = ColumnDataSource(hist_df)\n    plot = figure(plot_height=400, plot_width=600, title=title, x_axis_label=column, y_axis_label='image count')\n    plot.quad(bottom=0, top=column, left='left', right='right', source=src, fill_color=colors[0], line_color='#35838d', fill_alpha=0.7, hover_fill_alpha=0.7, hover_fill_color=colors[1])\n    #hover tool\n    hover = HoverTool(tooltips=[('Interval', '@interval'), ('img count', str(\"@\"+column))])\n    plot.add_tools(hover)\n    #output_file(f'{base_path}/wheat_spikes_per_img.html')\n    output_notebook()\n    show(plot)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_hover(train_img_count, 'num_bboxes', title='Number of wheat spikes per image')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# \"\"\"# Examples of images with small/large num of spikes\n# low_num_of_spikes = train_img_count[train_img_count.num_bboxes < 10].image_id\n# plt_img(train_images[train_images.image_id.isin(low_num_of_spikes)], title='Example of images with small number of spikes')\n# high_num_of_spikes = train_img_count[train_img_count.num_bboxes > 100].image_id\n# plt_img(train_images[train_images.image_id.isin(high_num_of_spikes)], title='Example of images with high number of spikes')\n# \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_hover(train_images, 'bbox_area', title='Area of one bbox')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(train_images.bbox_area.max())\n# \"\"\"Because the max area of a bbox is 529788.0 we want to check which image id have big bbox area and delete it\n#    Similarly we want to check the min bboxes area and delete it\"\"\"\nbig_bboxes = train_images[train_images.bbox_area > 180000].image_id # 180,000 = 220,000 = 5 images\n# plt_img(train_images[train_images.image_id.isin(big_bboxes)], title='Example of images with big bbox area')\n# print(big_bboxes)\n\nmin_area = train_images[train_images.bbox_area > 0].bbox_area.min()\n# print(min_area)\nsmall_bboxes = train_images[(train_images.bbox_area < 50) & (train_images.bbox_area > 0)].image_id # maybe change the 50 number\n# plt_img(train_images[train_images.image_id.isin(small_bboxes)], title='Example of images with small bbox area')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Area of bounding box per image\"\"\"\nbbox_area_per_img = train_images.groupby(by='image_id').sum().reset_index()\nbbox_percentage = bbox_area_per_img.copy()\nbbox_percentage.bbox_area = bbox_percentage.bbox_area/(1024 * 1024) * 100 # normalization of bbox area\nhist_hover(bbox_percentage, 'bbox_area', title='Percentage of image area covered by bboxes')\nprint(bbox_percentage.bbox_area.max()) # Max is bigger then 100% (108.19730758666992%) --> bboxes are overlapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bbox_delete(df):\n    for index, col in df.iterrows():\n        if col['bbox_area'] > 180000:\n            df.drop(index, axis=0, inplace=True)\n        # elif (col['bbox_area'] < 50) & (col['bbox_area'] > 0):\n        #     df.drop(index, axis=0, inplace=True)\n        elif (col['x_min'] == 0) & (col['y_min'] == 0) & (col['width'] == 0) & (col['height'] == 0):\n            df.drop(index, axis=0, inplace=True)\n        elif (col['x_max'] <= col['x_min']):\n            df.drop(index, axis=0, inplace=True)\n        elif (col['y_max'] <= col['y_min']):\n            df.drop(index, axis=0, inplace=True)\n        elif col['width'] > 400 or col['height'] > 400:\n            df.drop(index, axis=0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bbox_delete(train_images)\nlen(train_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_hover(train_images, 'width', title='Histogram of bbox width')\nhist_hover(train_images, 'height', title='Histogram of bbox height')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split the train data into train and validation sets (validation set is 15%)\nimages_ids = train_images.image_id.unique()\ntrain_ids = images_ids[:-510]\nvalid_ids = images_ids[-510:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dataframes from array\ntrain_df = train_images[train_images.image_id.isin(train_ids)]\nvalid_df = train_images[train_images.image_id.isin(valid_ids)]\ntrain_df_shape = train_df.shape  # (125689, 13)\nvalid_df_shape = valid_df.shape  # (22010, 13)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Creating the model\"\"\"\n\n\nclass WheatDataset(Dataset):\n    def __init__(self, dataframe, image_dir, transforms=None):\n        super().__init__()\n\n        self.image_ids = dataframe.image_id.unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)  # change the shape from [h,w,c] to [c,h,w]\n#         image = torch.from_numpy(image).permute(2,0,1)\n        image /= 255.0\n\n        records = self.df[self.df['image_id'] == image_id]\n        boxes = records[['x_min', 'y_min', 'width', 'height']].values\n#         boxes = records[['x_min', 'y_min', 'bbox_width', 'bbox_height']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        area = torch.as_tensor(area, dtype=torch.float32)\n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        \n\n        # there is only one class\n        labels = torch.ones((records.shape[0],), dtype=torch.int64)\n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((records.shape[0],), dtype=torch.int64)\n\n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        # target['masks'] = None\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        target['iscrowd'] = iscrowd\n        \n#         target['boxes'] = torch.zeros((0, 4)) if ('num_bboxes' != 0) else 'bbox' #added to avoid crop issue\n        # target = {'boxes': boxes, 'labels': labels, 'image_id': torch.tensor([index]), 'area': area, 'iscrowd': iscrowd}\n        # target['masks'] = None\n\n        if self.transforms:\n            sample = {'image': image, 'bboxes': target['boxes'], 'labels': labels}\n            sample = self.transforms(**sample)\n            image = sample['image']\n#             target['boxes'] = torch.tensor(sample['bboxes']).float()\n#             target['boxes'] = torch.tensor(sample['bboxes'])\n            target['boxes'] = target['boxes'].type(torch.float32)\n            # target['boxes'] = torch.tensor(sample['bboxes']).float()\n#             target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0).float()\n\n           \n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load a model; pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n\nnum_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nmodel.to(device)\n# 1 == 1  # checks which device is connected (cpu or gpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Averager:\n    def __init__(self):\n        self.current_total = 0.0\n        self.iterations = 0.0\n\n    def send(self, value):\n        self.current_total += value\n        self.iterations += 1\n\n    @property\n    def value(self):\n        if self.iterations == 0:\n            return 0\n        else:\n            return 1.0 * self.current_total / self.iterations\n\n    def reset(self):\n        self.current_total = 0.0\n        self.iterations = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Calculation of IoU\"\"\"\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float: #(0.0 <= IoU <= 1.0)\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n        \n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n        \n#         Overlap area calculation\n    deltax = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    if deltax < 0:\n        return 0.0\n    \n    deltay = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n    if deltay < 0:\n        return 0.0\n    \n    area_of_overlap = deltax * deltay\n    \n    union_area = (((gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1)) + ((pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1)) - area_of_overlap)\n    \n    return area_of_overlap / union_area","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"No overlap\"\"\"\n    \nbbox1 = np.array([834.0, 222.0, 56.0, 36.0])\nbbox2 = np.array([26.0, 144.0, 124.0, 117.0])\n    \nassert calculate_iou(bbox1, bbox2, form='coco') == 0\n\n\"\"\"Partial overlap\"\"\"\n\nbbox1 = np.array([100, 100, 100, 100])\nbbox2 = np.array([100, 100, 200, 100])\n\nres = calculate_iou(bbox1, bbox2, form='coco')\nassert  res > 0.5 and res < 0.50249\n\n\"\"\"Full overlap\"\"\"\nbbox1 = np.array([834.0, 222.0, 56.0, 36.0])\nbbox2 = bbox1\n\nassert calculate_iou(bbox1, bbox2, form='coco') == 1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Returns the index of the highest IoU between the ground-truth boxes and the prediction\"\"\"\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n    \n    for gt_idx in range(len(gts)):\n        if gts[gt_idx][0] < 0: #matches to GT-bbox\n            continue \n            \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n        \n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Calculates precision for GT - prediction pairs at one threshold\"\"\"\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    '''    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision'''\n    k = len(preds)\n    fp = 0 #False positive\n    tp = 0 #True positive\n    \n    \n    for prediction_idx in range(k): #pred in enumerate(preds_sorted):\n\n        gt_idx_highest_iou = find_best_match(gts, preds[prediction_idx], prediction_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if gt_idx_highest_iou >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[gt_idx_highest_iou] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n    \n'''Calculation of image precision'''\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form='coco') -> float:\n    '''    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision'''\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data augmentation visualization for testing\n\nexample_transforms = albu.Compose([\n    albu.RandomResizedCrop(height=256, width=256, scale=(0.08, 1.0), ratio=(1, 1), p=0.5),\n    albu.HorizontalFlip(p=0.5),\n    albu.VerticalFlip(p=0.5),\n    albu.ToSepia(),     \n    albu.OneOf([albu.RGBShift(),\n                albu.HueSaturationValue(),\n                albu.RandomGamma(),\n                albu.RandomBrightness()], p=1.0),\n    albu.CLAHE(p=0.5)\n], p=1.0, bbox_params=albu.BboxParams(format='coco', min_visibility=0.3, label_fields=['category_id']))\n\ndef apply_transforms(transforms, df, n_transforms=3):\n    idx = np.random.randint(len(df), size=1)[0]\n    \n    image_id = df.iloc[idx].image_id\n    bboxes = []\n    for _, row in df[df.image_id == image_id].iterrows():\n        bboxes.append([row.x_min, row.y_min, row.width, row.height])\n    \n    image = Image.open(train_dir + image_id + '.jpg')\n    \n    fig, axs = plt.subplots(1, n_transforms+1, figsize=(15,7))\n    \n    # plot the original image\n    axs[0].imshow(image)\n    axs[0].set_title('original - ' + image_id)\n    for bbox in bboxes:\n        rect = patches.Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=1,edgecolor='r',facecolor='none')\n        axs[0].add_patch(rect)\n    \n    # apply transforms n_transforms times\n    for i in range(n_transforms):\n        params = {'image': np.asarray(image),\n                  'bboxes': bboxes,\n                  'category_id': [1 for j in range(len(bboxes))]}\n        augmented_boxes = transforms(**params)\n        bboxes_aug = augmented_boxes['bboxes']\n        image_aug = augmented_boxes['image']\n\n        # plot the augmented image and augmented bounding boxes\n        axs[i+1].imshow(image_aug)\n        axs[i+1].set_title('augmented_' + str(i+1))\n        for bbox in bboxes_aug:\n            rect = patches.Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=1,edgecolor='r',facecolor='none')\n            axs[i+1].add_patch(rect)\n    plt.show()\n\napply_transforms(example_transforms, train_df, n_transforms=3)\n\n\n\n# Albumentations - bbox safe functions for data \n# All those commented out caused extreme problems during training\n\ndef train_transforms():\n    return albu.Compose([\n#     albu.RandomSizedBBoxSafeCrop(height=256, width=256, p=0.5),\n    albu.HorizontalFlip(p=0.5),\n    albu.VerticalFlip(p=0.5),\n    albu.ToSepia(),     \n    albu.OneOf([\n#                 albu.RGBShift(),\n#                 albu.HueSaturationValue(),\n                albu.RandomGamma(),\n                albu.RandomBrightness()], p=1.0),\n#     albu.CLAHE(p=0.5),\n    ToTensorV2(p=1.0)],\n    bbox_params={'format': 'pascal_voc', 'min_visibility':0.3, 'label_fields': ['labels']})\n\n# For testing without augmentation\ndef get_train_transform():\n    return albu.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\n# Validation set never gets augmented\ndef get_valid_transform():\n    return albu.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# idx = np.random.randint(len(train_df), size=1)[0]\n    \n# image_id = train_df.iloc[idx].image_id\n# bboxes = []\n# for _, row in train_df[train_df.image_id == image_id].iterrows():\n#     bboxes.append([row.x_min, row.y_min, row.width, row.height])\n           \n# image = Image.open(train_dir + image_id + '.jpg')\n# transformed = example_transforms(image='image', bboxes='bboxes')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transform():\n    return albu.Compose([\n        albu.Flip(0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transform():\n    return albu.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ntrain_dataset = WheatDataset(train_df, train_dir, train_transforms())\nvalid_dataset = WheatDataset(valid_df, train_dir, get_valid_transform())\n# test_dataset = WheatTestDataset(test_df, DIR_TEST, get_test_transform())\n# indices = torch.randperm(len(train_dataset)).tolist()#splits dataset into train and val\n\ntrain_data_loader = DataLoader(train_dataset, batch_size=16, num_workers=4, shuffle=True, collate_fn=collate_fn)\nvalid_data_loader = DataLoader(valid_dataset, batch_size=8, num_workers=4, shuffle=True, collate_fn=collate_fn)\n# test_data_loader = DataLoader(test_dataset, batch_size=4, shuffle=False, num_workers=4, drop_last=False, collate_fn=collate_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Valid\"\"\"\nvalid_batch = next(iter(valid_data_loader))\nimages, targets, image_ids = valid_batch\nimages = list(img.to(device) for img in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\nboxes = targets[1]['boxes'].cpu().numpy().astype(np.int32)\n# image_ids = np.array(image_ids)\nsample = images[1].permute(1,2,0).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# boxes = targets[2]['boxes'].cpu().numpy().asarray()\nboxes = targets[2]['boxes'].cpu().numpy().astype(np.float32)\n# sample = images[2].cpu().numpy()\nsample = images[2].permute(1,2,0).cpu().numpy()\n\nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n\nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Train\"\"\"\nmodel.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.005, momentum=0.85, weight_decay=0.0005)\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n# lr_scheduler = None\nnum_epochs = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_hist = Averager()\n# score_hist = Averager()\ndetection_threshold = 0.5\nloss_values = []\n\nitr = 1\n\nfor epoch in range(num_epochs):\n    loss_hist.reset()\n    running_loss = 0.0\n#     score_hist.reset()\n    \n    for images, targets, image_id in train_data_loader:\n\n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n        model.train()\n        loss_dict = model(images, targets)\n\n        losses = sum(loss for loss in loss_dict.values())\n        loss_value = losses.item()\n        running_loss =+ loss_value * len(images)\n        loss_hist.send(loss_value)\n\n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n        lr_scheduler.step()\n      \n\n        \n        if itr % 50 == 0:\n            print(f\"Iteration #{itr} train loss: {loss_value}\")\n#         loss_values.append(running_loss / len(train_dataset))\n        \n        itr += 1 \n\n    # update the learning rate\n    if lr_scheduler is not None:\n        lr_scheduler.step()\n        \n    print(f\"Epoch #{epoch} Loss: {loss_hist.value}\")\n    \n    for i, data in enumerate(train_data_loader, 0):\n        running_loss =+ loss_value * len(images)\n    loss_values.append(running_loss / len(train_dataset))\n\n    with torch.no_grad():\n\n        validation_image_precisions = []\n        iou_thresholds = [x for x in np.arange(0.5, 0.76, 0.05)]\n\n        for images, targets, image_ids in valid_data_loader:\n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n            \n            model.eval()\n            outputs = model(images, targets)\n\n#         outputs = model.forward(images)\n        # validation losses\n\n        # Calculate validation losses\n#         loss_dict = model(images, targets)\n#         losses = sum(loss for loss in loss_dict.values())\n#         loss_value = losses.item()\n\n\n        # validation score\n\n            # Calculating mAP@\n            for i, image in enumerate(images):\n                boxes = outputs[i]['boxes'].data.cpu().numpy()\n                scores = outputs[i]['scores'].data.cpu().numpy()\n#                 boxes = boxes[scores >= detection_threshold].astype(np.int32)\n                target = targets[i]['boxes'].cpu().data.numpy()\n    #             sort_pred_idx = np.argsort(scores)[::-1]\n    #             sort_pred = boxes[sort_pred_idx]\n                    # shape is x1,y1,x2,y2 (pascal_voc)\n                image_precision = calculate_image_precision(target,\n                                                        boxes,\n                                                        thresholds=iou_thresholds,\n                                                        form='pascal_voc')\n\n                validation_image_precisions.append(image_precision)\n    #         if itr % 50 == 0:\n    #             print(f\"Iteration #{itr} loss: {loss_value}\")\n    #             print(f\"Iteration #{itr} train loss: {loss_value}\")\n    #             print(f\"Iteration #{itr} score: {np.mean(validation_image_precisions)}\")\n\n    #         itr += 1\n\n    print(f\"Validation #{epoch} score: {np.mean(validation_image_precisions)}\")\n\n    print(\"Validation IOU: {0:.4f}\".format(np.mean(validation_image_precisions)))\nplt.plot(loss_values)\n\n\"\"\"If we need to calculate validation loss (i think we don't need to cause we don't train the validation??)\n#             model.train()\n#                 # Calculate validation losses\n#             loss_dict = model(images, targets)\n#             losses = sum(loss for loss in loss_dict.values())\n#             loss_value = losses.item()\n##             output = model(images, targets)\n# #            loss = criterion(output, target)\n              running_loss =+ loss_value * len(images)\n              loss_hist.send(loss_value)\n\n              optimizer.zero_grad()\n              losses.backward()\n              optimizer.step()\n              lr_scheduler.step()\n\n#             if itr % 50 == 0:\n#                 print(f\"Validation loss #{epoch} score: {np.mean(validation_image_precisions)}\")\n\n#             itr += 1 \n\n#         print(f\"Validation loss #{epoch} score: {np.mean(validation_image_precisions)}\")\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), 'fasterrcnn_resnet50_fpn.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Our testing sample\nsample_id = '1ef16dab1'\n\ngt_boxes = train_df[train_df['image_id'] == sample_id][['x_min', 'y_min', 'width', 'height']].values\ngt_boxes = gt_boxes.astype(np.int)\n\n# Ground-truth boxes of our sample\ngt_boxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No GT - Predicted box match\npred_box = np.array([0, 0, 20, 10])\nassert find_best_match(gt_boxes, pred_box, 0, threshold=0.5, form='coco') == -1\n\n# First GT match\npred_box = np.array([954., 391., 70., 90.])\nassert find_best_match(gt_boxes, pred_box, 0, threshold=0.5, form='coco') == 0\n\n# These are the predicted boxes (and scores) from my locally trained model.\npreds = np.array([[956, 409, 68, 85],\n                  [883, 945, 85, 77],\n                  [745, 468, 81, 87],\n                  [658, 239, 103, 105],\n                  [518, 419, 91, 100],\n                  [711, 805, 92, 106],\n                  [62, 213, 72, 64],\n                  [884, 175, 109, 68],\n                  [721, 626, 96, 104],\n                  [878, 619, 121, 81],\n                  [887, 107, 111, 71],\n                  [827, 525, 88, 83],\n                  [816, 868, 102, 86],\n                  [166, 882, 78, 75],\n                  [603, 563, 78, 97],\n                  [744, 916, 68, 52],\n                  [582, 86, 86, 72],\n                  [79, 715, 91, 101],\n                  [246, 586, 95, 80],\n                  [181, 512, 93, 89],\n                  [655, 527, 99, 90],\n                  [568, 363, 61, 76],\n                  [9, 717, 152, 110],\n                  [576, 698, 75, 78],\n                  [805, 974, 75, 50],\n                  [10, 15, 78, 64],\n                  [826, 40, 69, 74],\n                  [32, 983, 106, 40]]\n                )\n\nscores = np.array([0.9932319, 0.99206185, 0.99145633, 0.9898089, 0.98906296, 0.9817738,\n                   0.9799762, 0.97967803, 0.9771589, 0.97688967, 0.9562935, 0.9423076,\n                   0.93556845, 0.9236257, 0.9102379, 0.88644403, 0.8808225, 0.85238415,\n                   0.8472188, 0.8417798, 0.79908705, 0.7963756, 0.7437897, 0.6044758,\n                   0.59249884, 0.5557045, 0.53130984, 0.5020239])\n\n\n# Sort highest confidence -> lowest confidence\npreds_sorted_idx = np.argsort(scores)[::-1]\npreds_sorted = preds[preds_sorted_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_result(sample_id, preds, gt_boxes):\n    sample = cv2.imread(f'{train_dir}/{sample_id}.jpg', cv2.IMREAD_COLOR)\n    sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for pred_box in preds:\n        cv2.rectangle(\n            sample,\n            (pred_box[0], pred_box[1]),\n            (pred_box[0] + pred_box[2], pred_box[1] + pred_box[3]),\n            (220, 0, 0), 2\n        )\n\n    for gt_box in gt_boxes:    \n        cv2.rectangle(\n            sample,\n            (gt_box[0], gt_box[1]),\n            (gt_box[0] + gt_box[2], gt_box[1] + gt_box[3]),\n            (0, 220, 0), 2\n        )\n# cv2.putText(image, \"IoU: {:.4f}\".format(iou), (10, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)\n# ax.putText(image, \"IoU: {:.4f}\".format(iou), (10, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)\n    ax.set_axis_off()\n    ax.imshow(sample)\n    ax.set_title(\"RED: Predicted | GREEN - Ground-truth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_result(sample_id, preds, gt_boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n\nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(f'{base_path}/sample_submission.csv')\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_transform():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load a model; pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\nnum_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n# Load the trained weights\nmodel.load_state_dict(torch.load(weights_file))\nmodel.eval()\n\ntest = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"detection_threshold = 0.5\nresults = []\n\nfor images, image_ids in test_data_loader:\n\n    images = list(image.to(device) for image in images)\n    outputs = model(images)\n\n    for i, image in enumerate(images):\n\n        boxes = outputs[i]['boxes'].data.cpu().numpy()\n        scores = outputs[i]['scores'].data.cpu().numpy()\n        \n        boxes = boxes[scores >= detection_threshold].astype(np.int32)\n        scores = scores[scores >= detection_threshold]\n        image_id = image_ids[i]\n        \n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n        \n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n\n        \n        results.append(result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results[0:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = images[1].permute(1,2,0).cpu().numpy()\nboxes = outputs[1]['boxes'].data.cpu().numpy()\nscores = outputs[1]['scores'].data.cpu().numpy()\n\nboxes = boxes[scores >= detection_threshold].astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 2)\n    \nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}