{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h1> EDA ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# import useful tools\nfrom glob import glob\nfrom PIL import Image\nimport cv2\n\n# import data visualization\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\n\nfrom bokeh.plotting import figure\nfrom bokeh.io import output_notebook, show, output_file\nfrom bokeh.models import ColumnDataSource, HoverTool, Panel\nfrom bokeh.models.widgets import Tabs\n\n# import data augmentation\nimport albumentations as A","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR = '/kaggle/input/global-wheat-detection/train/'\nTEST_DIR = '/kaggle/input/global-wheat-detection/test/'\ntrain_dir_csv = '/kaggle/input/global-wheat-detection/train.csv'\n\nTRAIN_DIR_G = glob(TRAIN_DIR + '*')\nTEST_DIR_G = glob(TEST_DIR + '*')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('The length of train data {}'.format(len(TRAIN_DIR_G)))\nprint('The length of test data {}'.format(len(TEST_DIR_G)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(train_dir_csv)\n\nall_train_images = pd.DataFrame([fns.split('/')[-1][:-4] for fns in TRAIN_DIR_G ])\nall_train_images.columns = ['image_id']\n\nall_train_images = all_train_images.merge(train, on = 'image_id', how = 'left')\nall_train_images['bbox'] = all_train_images.bbox.fillna('[0,0,0,0]')\n\nbbox_items = all_train_images.bbox.str.split(',', expand=True)\nall_train_images['bbox_xmin'] = bbox_items[0].str.strip('[ ').astype(np.float)\nall_train_images['bbox_ymin'] = bbox_items[1].str.strip(' ').astype(np.float)\nall_train_images['bbox_width'] = bbox_items[2].str.strip(' ').astype(np.float)\nall_train_images['bbox_height'] = bbox_items[3].str.strip(' ]').astype(np.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images['bbox_area'] = all_train_images['bbox_width'] * all_train_images['bbox_height']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dropped empty boxes before so i can drop images without boxes now\nprint('{} images without wheat heads.'.format(len(all_train_images) - len(train)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#examples with big bounding boxes that will be dropped \nlarge_boxes_ids_try = all_train_images[all_train_images['bbox_area'] > 150000].image_id\nlarge_boxes_ids_try","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images = all_train_images.drop(large_boxes_ids_try.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removing images without bboxes\nall_train_images = all_train_images[~(all_train_images['bbox_area']==0)] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_brightness(image):\n    # convert to grayscale\n    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    \n    # get average brightness\n    return np.array(gray).mean()\n\ndef add_brightness(df):\n    brightness = []\n    for _, row in df.iterrows():\n        img_id = row.image_id  \n        image = cv2.imread(TRAIN_DIR + img_id + '.jpg')\n        brightness.append(get_image_brightness(image))\n        \n    brightness_df = pd.DataFrame(brightness)\n    brightness_df.columns = ['brightness']\n    df = pd.concat([df, brightness_df], ignore_index=True, axis=1)\n    df.columns = ['image_id', 'brightness']\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add brightness to the dataframe\nimages_df = pd.DataFrame(all_train_images.image_id.unique())\nimages_df.columns = ['image_id']\nbrightness_df = add_brightness(images_df)\n\nall_train_images = all_train_images.merge(brightness_df, on='image_id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1> Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom collections import namedtuple\n\nimport torch\nimport torchvision\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.utils.data as data_utils\n\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\n\nfrom torch.utils.data.sampler import SequentialSampler\n\nfrom matplotlib import pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations.core.transforms_interface import DualTransform\n\nfrom albumentations.augmentations.bbox_utils import denormalize_bbox, normalize_bbox\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom matplotlib import pyplot as plt\nimport matplotlib.patches as patches\nfrom matplotlib.image import imsave\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images['bbox_xmin'] = all_train_images['bbox_xmin'].astype(np.float)\nall_train_images['bbox_ymin'] = all_train_images['bbox_ymin'].astype(np.float)\nall_train_images['bbox_width'] = all_train_images['bbox_width'].astype(np.float)\nall_train_images['bbox_height'] = all_train_images['bbox_height'].astype(np.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WheatDataset(Dataset):\n\n    def __init__(self, dataframe, image_dir, transforms=None):\n        super().__init__()\n\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n\n        boxes = records[['bbox_xmin', 'bbox_ymin', 'bbox_width', 'bbox_height']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        \n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        area = torch.as_tensor(area, dtype=torch.float32)\n\n        # there is only one class\n        labels = torch.ones((records.shape[0],), dtype=torch.int64)\n        \n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((records.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        # target['masks'] = None\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        target['iscrowd'] = iscrowd\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            \n            target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomCutout(DualTransform):\n    \"\"\"\n    Custom Cutout augmentation with handling of bounding boxes \n    Note: (only supports square cutout regions)\n    \n    Author: Kaushal28\n    Reference: https://arxiv.org/pdf/1708.04552.pdf\n    \"\"\"\n    \n    def __init__(\n        self,\n        fill_value=0,\n        bbox_removal_threshold=0.50,\n        min_cutout_size=192,\n        max_cutout_size=512,\n        always_apply=False,\n        p=0.5\n    ):\n        \"\"\"\n        Class construstor\n        \n        :param fill_value: Value to be filled in cutout (default is 0 or black color)\n        :param bbox_removal_threshold: Bboxes having content cut by cutout path more than this threshold will be removed\n        :param min_cutout_size: minimum size of cutout (192 x 192)\n        :param max_cutout_size: maximum size of cutout (512 x 512)\n        \"\"\"\n        super(CustomCutout, self).__init__(always_apply, p)  # Initialize parent class\n        self.fill_value = fill_value\n        self.bbox_removal_threshold = bbox_removal_threshold\n        self.min_cutout_size = min_cutout_size\n        self.max_cutout_size = max_cutout_size\n        \n    def _get_cutout_position(self, img_height, img_width, cutout_size):\n        \"\"\"\n        Randomly generates cutout position as a named tuple\n        \n        :param img_height: height of the original image\n        :param img_width: width of the original image\n        :param cutout_size: size of the cutout patch (square)\n        :returns position of cutout patch as a named tuple\n        \"\"\"\n        position = namedtuple('Point', 'x y')\n        return position(\n            np.random.randint(0, img_width - cutout_size + 1),\n            np.random.randint(0, img_height - cutout_size + 1)\n        )\n        \n    def _get_cutout(self, img_height, img_width):\n        \"\"\"\n        Creates a cutout pacth with given fill value and determines the position in the original image\n        \n        :param img_height: height of the original image\n        :param img_width: width of the original image\n        :returns (cutout patch, cutout size, cutout position)\n        \"\"\"\n        cutout_size = np.random.randint(self.min_cutout_size, self.max_cutout_size + 1)\n        cutout_position = self._get_cutout_position(img_height, img_width, cutout_size)\n        return np.full((cutout_size, cutout_size, 3), self.fill_value), cutout_size, cutout_position\n        \n    def apply(self, image, **params):\n        \"\"\"\n        Applies the cutout augmentation on the given image\n        \n        :param image: The image to be augmented\n        :returns augmented image\n        \"\"\"\n        image = image.copy()  # Don't change the original image\n        self.img_height, self.img_width, _ = image.shape\n        cutout_arr, cutout_size, cutout_pos = self._get_cutout(self.img_height, self.img_width)\n        \n        # Set to instance variables to use this later\n        self.image = image\n        self.cutout_pos = cutout_pos\n        self.cutout_size = cutout_size\n        \n        image[cutout_pos.y:cutout_pos.y+cutout_size, cutout_pos.x:cutout_size+cutout_pos.x, :] = cutout_arr\n        return image\n    \n    def apply_to_bbox(self, bbox, **params):\n        \"\"\"\n        Removes the bounding boxes which are covered by the applied cutout\n        \n        :param bbox: A single bounding box coordinates in pascal_voc format\n        :returns transformed bbox's coordinates\n        \"\"\"\n\n        # Denormalize the bbox coordinates\n        bbox = denormalize_bbox(bbox, self.img_height, self.img_width)\n        x_min, y_min, x_max, y_max = tuple(map(int, bbox))\n\n        bbox_size = (x_max - x_min) * (y_max - y_min)  # width * height\n        overlapping_size = np.sum(\n            (self.image[y_min:y_max, x_min:x_max, 0] == self.fill_value) &\n            (self.image[y_min:y_max, x_min:x_max, 1] == self.fill_value) &\n            (self.image[y_min:y_max, x_min:x_max, 2] == self.fill_value)\n        )\n\n        # Remove the bbox if it has more than some threshold of content is inside the cutout patch\n        if overlapping_size / bbox_size > self.bbox_removal_threshold:\n            return normalize_bbox((0, 0, 0, 0), self.img_height, self.img_width)\n\n        return normalize_bbox(bbox, self.img_height, self.img_width)\n\n    def get_transform_init_args_names(self):\n        \"\"\"\n        Fetches the parameter(s) of __init__ method\n        :returns: tuple of parameter(s) of __init__ method\n        \"\"\"\n        return ('fill_value', 'bbox_removal_threshold', 'min_cutout_size', 'max_cutout_size', 'always_apply', 'p')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augmentations\ndef get_train_transform():\n    return A.Compose([\n        #A.Resize(512,512),\n        #CustomCutout(p=1),\n        #A.Flip(p=0.5),\n        #A.Rotate(limit=90, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\n        #A.Equalize(mode='cv', by_channels=False, mask=None, mask_params=(), always_apply=False, p=0.5),\n        #A.OneOf([  # One of blur or adding gauss noise\n        #A.Blur(p=0.5),  # Blurs the image\n        #A.GaussNoise(var_limit=(5.0 / 255.0, 25.0 / 255.0), mean = 15.0/255.0, p=0.5)  # Adds Gauss noise to image\n   # ], p=0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transform():\n    return A.Compose([\n        #A.Resize(512,512),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1> Creating model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Averager:\n    def __init__(self):\n        self.current_total = 0.0\n        self.iterations = 0.0\n\n    def send(self, value):\n        self.current_total += value\n        self.iterations += 1\n\n    @property\n    def value(self):\n        if self.iterations == 0:\n            return 0\n        else:\n            return 1.0 * self.current_total / self.iterations\n\n    def reset(self):\n        self.current_total = 0.0\n        self.iterations = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_ids = all_train_images['image_id'].unique()\nvalid_ids = image_ids[-665:]\ntrain_ids = image_ids[:-665]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = all_train_images[all_train_images['image_id'].isin(valid_ids)]\ntrain_df = all_train_images[all_train_images['image_id'].isin(train_ids)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df.shape, train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_dataset = WheatDataset(train_df, TRAIN_DIR, get_train_transform())\nvalid_dataset = WheatDataset(valid_df, TRAIN_DIR, get_valid_transform())\n\n\n# split the dataset in train and test set\nindices = torch.randperm(len(train_dataset)).tolist()\n\ntrain_data_loader = DataLoader(\n    train_dataset,\n    batch_size=8,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=16,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, targets, image_ids = next(iter(train_data_loader))\nimages = list(image.to(device) for image in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes = targets[2]['boxes'].cpu().numpy().astype(np.int32)\nsample = images[2].permute(1,2,0).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    \nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1> Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images['bbox_xmin'] = all_train_images['bbox_xmin'].astype(np.double)\nall_train_images['bbox_ymin'] = all_train_images['bbox_ymin'].astype(np.double)\nall_train_images['bbox_width'] = all_train_images['bbox_width'].astype(np.double)\nall_train_images['bbox_height'] = all_train_images['bbox_height'].astype(np.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(all_train_images['image_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\nlr_scheduler = None\n\nnum_epochs = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_hist = Averager()\nitr = 1\n\nfor epoch in range(num_epochs):\n    loss_hist.reset()\n    \n    for images, targets, image_ids in train_data_loader:\n        \n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n        loss_dict = model(images, targets)\n\n        losses = sum(loss for loss in loss_dict.values())\n        loss_value = losses.item()\n\n        loss_hist.send(loss_value)\n\n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n\n        if itr % 50 == 0:\n            print(f\"Iteration #{itr} loss: {loss_value}\")\n\n        itr += 1\n    \n    # update the learning rate\n    if lr_scheduler is not None:\n        lr_scheduler.step()\n\n    print(f\"Epoch #{epoch} loss: {loss_hist.value}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, targets, image_ids = next(iter(valid_data_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = list(img.to(device) for img in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes = targets[1]['boxes'].cpu().numpy().astype(np.int32)\nsample = images[1].permute(1,2,0).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\ncpu_device = torch.device(\"cpu\")\n\noutputs = model(images)\noutputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    \nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1> Saving the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), 'fasterrcnn_resnet50_fpn.pth')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}