{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### I've created this notebook as this is my first object detection challenge and thus wanted to explore image augmentation when each image has multiple bounding boxes and labels. Sharing this publicly as this might be useful to beginners like me.","metadata":{}},{"cell_type":"code","source":"import os\nimport ast\nfrom collections import namedtuple\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport joblib\nfrom joblib import Parallel, delayed\n\nimport cv2\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations.core.transforms_interface import DualTransform\nfrom albumentations.augmentations.bbox_utils import denormalize_bbox, normalize_bbox\n\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.utils.data as data_utils\n\nfrom matplotlib import pyplot as plt\nimport matplotlib.patches as patches\nfrom matplotlib.image import imsave","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-11T16:03:01.330940Z","iopub.execute_input":"2023-05-11T16:03:01.331452Z","iopub.status.idle":"2023-05-11T16:03:04.975885Z","shell.execute_reply.started":"2023-05-11T16:03:01.331392Z","shell.execute_reply":"2023-05-11T16:03:04.974489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nBASE_DIR = '/kaggle/input/global-wheat-detection'\nWORK_DIR = '/kaggle/working'\nBATCH_SIZE = 16\n\n# Set seed for numpy for reproducibility\nnp.random.seed(1996)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(BASE_DIR, 'train.csv'))\n\n# Let's expand the bounding box coordinates and calculate the area of all the bboxes\ntrain_df[['x_min','y_min', 'width', 'height']] = pd.DataFrame([ast.literal_eval(x) for x in train_df.bbox.tolist()], index= train_df.index)\ntrain_df = train_df[['image_id', 'bbox', 'source', 'x_min', 'y_min', 'width', 'height']]\ntrain_df['area'] = train_df['width'] * train_df['height']\ntrain_df['x_max'] = train_df['x_min'] + train_df['width']\ntrain_df['y_max'] = train_df['y_min'] + train_df['height']\ntrain_df = train_df.drop(['bbox'], axis=1)\ntrain_df = train_df[['image_id', 'x_min', 'y_min', 'x_max', 'y_max', 'width', 'height', 'area', 'source']]\n\n# There are some buggy annonations in training images having huge bounding boxes. Let's remove those bboxes\ntrain_df = train_df[train_df['area'] < 100000]\n\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = train_df['image_id'].unique()\nprint(f'Total number of training images: {len(image_ids)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are two major formats of bounding boxes:\n\n1. **pascal_voc**, which is [x_min, y_min, x_max, y_max]\n2. **COCO**, which is [x_min, y_min, width, height]\n\nWe'll see how to perform image augmentations for both the formats. Let's first start with **pascal_voc** format.","metadata":{}},{"cell_type":"code","source":"# Read the image on which data augmentaion is to be performed\nimage_id = 'c14c1e300'\nimage = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\nimage /= 255.0\nplt.figure(figsize = (10, 10))\nplt.imshow(image)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We've our image ready, let's create an array of bounding boxes for all the wheat heads in the above image and the array of labels (we've only 2 class here: wheat head and background). As all bounding boxes are of same class, labels array will contain only 1's. ","metadata":{}},{"cell_type":"code","source":"pascal_voc_boxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'x_max', 'y_max']].astype(np.int32).values\ncoco_boxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'width', 'height']].astype(np.int32).values\nassert(len(pascal_voc_boxes) == len(coco_boxes))\nlabels = np.ones((len(pascal_voc_boxes), ))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's plot the bounding boxes on the above image","metadata":{}},{"cell_type":"code","source":"def get_bbox(bboxes, col, color='white', bbox_format='pascal_voc'):\n    \n    for i in range(len(bboxes)):\n        # Create a Rectangle patch\n        if bbox_format == 'pascal_voc':\n            rect = patches.Rectangle(\n                (bboxes[i][0], bboxes[i][1]),\n                bboxes[i][2] - bboxes[i][0], \n                bboxes[i][3] - bboxes[i][1], \n                linewidth=2, \n                edgecolor=color, \n                facecolor='none')\n        else:\n            rect = patches.Rectangle(\n                (bboxes[i][0], bboxes[i][1]),\n                bboxes[i][2], \n                bboxes[i][3], \n                linewidth=2, \n                edgecolor=color, \n                facecolor='none')\n\n        # Add the patch to the Axes\n        col.add_patch(rect)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's define augmentations using albumentations library. We will define a single verticle flip with probability 1 for re-producibility.","metadata":{}},{"cell_type":"code","source":"aug = albumentations.Compose([\n        albumentations.Resize(512, 512),   # Resize the given 1024 x 1024 image to 512 * 512\n        albumentations.VerticalFlip(1),    # Verticlly flip the image\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aug_result = aug(image=image, bboxes=pascal_voc_boxes, labels=labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16, 16))\nget_bbox(pascal_voc_boxes, ax[0], color='red')\nax[0].title.set_text('Original Image')\nax[0].imshow(image)\n\nget_bbox(aug_result['bboxes'], ax[1], color='red')\nax[1].title.set_text('Augmented Image')\nax[1].imshow(aug_result['image'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As you can see in above images, we resized the image to half of the original size and also vertically flipped the image. All the bounding boxes are also transformed accordingly.  ","metadata":{}},{"cell_type":"markdown","source":"Let's now do the same with COCO bounding box format.","metadata":{}},{"cell_type":"code","source":"aug = albumentations.Compose([\n        albumentations.Resize(512, 512),   # Resize the given 1024 x 1024 image to 512 * 512\n        albumentations.VerticalFlip(1),     # Verticlly flip the image\n        albumentations.Blur(p=1)\n    ], bbox_params={'format': 'coco', 'label_fields': ['labels']})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aug_result = aug(image=image, bboxes=coco_boxes, labels=labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16, 16))\nget_bbox(coco_boxes, ax[0], color='red', bbox_format='coco')\nax[0].title.set_text('Original Image')\nax[0].imshow(image)\n\nget_bbox(aug_result['bboxes'], ax[1], color='red', bbox_format='coco')\nax[1].title.set_text('Augmented Image')\nax[1].imshow(aug_result['image'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Edit: After learning a few augmentations techniques with albumentations, I'm implementing custom random cutout augmentation with handling of bounding boxes.","metadata":{}},{"cell_type":"markdown","source":"Note: We will use original image size and poscal VOC format for bboxes. Also will create a augmented dataset with new images instead of augmenting images on the fly while training.","metadata":{}},{"cell_type":"markdown","source":"## Let's define new augmentations\n\nUsing augmentation library we will perform following augmentations (which also takes care of bounding boxes coordinates)\n* Horizontal Flip\n* Vertical Flip\n* Blur\n* Adding GaussNoise\n* Cutout (Custom implementation to handle bounding boxes)","metadata":{}},{"cell_type":"code","source":"class CustomCutout(DualTransform):\n    \"\"\"\n    Custom Cutout augmentation with handling of bounding boxes \n    Note: (only supports square cutout regions)\n    \n    Author: Kaushal28\n    Reference: https://arxiv.org/pdf/1708.04552.pdf\n    \"\"\"\n    \n    def __init__(\n        self,\n        fill_value=0,\n        bbox_removal_threshold=0.50,\n        min_cutout_size=192,\n        max_cutout_size=512,\n        always_apply=False,\n        p=0.5\n    ):\n        \"\"\"\n        Class construstor\n        \n        :param fill_value: Value to be filled in cutout (default is 0 or black color)\n        :param bbox_removal_threshold: Bboxes having content cut by cutout path more than this threshold will be removed\n        :param min_cutout_size: minimum size of cutout (192 x 192)\n        :param max_cutout_size: maximum size of cutout (512 x 512)\n        \"\"\"\n        super(CustomCutout, self).__init__(always_apply, p)  # Initialize parent class\n        self.fill_value = fill_value\n        self.bbox_removal_threshold = bbox_removal_threshold\n        self.min_cutout_size = min_cutout_size\n        self.max_cutout_size = max_cutout_size\n        \n    def _get_cutout_position(self, img_height, img_width, cutout_size):\n        \"\"\"\n        Randomly generates cutout position as a named tuple\n        \n        :param img_height: height of the original image\n        :param img_width: width of the original image\n        :param cutout_size: size of the cutout patch (square)\n        :returns position of cutout patch as a named tuple\n        \"\"\"\n        position = namedtuple('Point', 'x y')\n        return position(\n            np.random.randint(0, img_width - cutout_size + 1),\n            np.random.randint(0, img_height - cutout_size + 1)\n        )\n        \n    def _get_cutout(self, img_height, img_width):\n        \"\"\"\n        Creates a cutout pacth with given fill value and determines the position in the original image\n        \n        :param img_height: height of the original image\n        :param img_width: width of the original image\n        :returns (cutout patch, cutout size, cutout position)\n        \"\"\"\n        cutout_size = np.random.randint(self.min_cutout_size, self.max_cutout_size + 1)\n        cutout_position = self._get_cutout_position(img_height, img_width, cutout_size)\n        return np.full((cutout_size, cutout_size, 3), self.fill_value), cutout_size, cutout_position\n        \n    def apply(self, image, **params):\n        \"\"\"\n        Applies the cutout augmentation on the given image\n        \n        :param image: The image to be augmented\n        :returns augmented image\n        \"\"\"\n        image = image.copy()  # Don't change the original image\n        self.img_height, self.img_width, _ = image.shape\n        cutout_arr, cutout_size, cutout_pos = self._get_cutout(self.img_height, self.img_width)\n        \n        # Set to instance variables to use this later\n        self.image = image\n        self.cutout_pos = cutout_pos\n        self.cutout_size = cutout_size\n        \n        image[cutout_pos.y:cutout_pos.y+cutout_size, cutout_pos.x:cutout_size+cutout_pos.x, :] = cutout_arr\n        return image\n    \n    def apply_to_bbox(self, bbox, **params):\n        \"\"\"\n        Removes the bounding boxes which are covered by the applied cutout\n        \n        :param bbox: A single bounding box coordinates in pascal_voc format\n        :returns transformed bbox's coordinates\n        \"\"\"\n\n        # Denormalize the bbox coordinates\n        bbox = denormalize_bbox(bbox, self.img_height, self.img_width)\n        x_min, y_min, x_max, y_max = tuple(map(int, bbox))\n\n        bbox_size = (x_max - x_min) * (y_max - y_min)  # width * height\n        overlapping_size = np.sum(\n            (self.image[y_min:y_max, x_min:x_max, 0] == self.fill_value) &\n            (self.image[y_min:y_max, x_min:x_max, 1] == self.fill_value) &\n            (self.image[y_min:y_max, x_min:x_max, 2] == self.fill_value)\n        )\n\n        # Remove the bbox if it has more than some threshold of content is inside the cutout patch\n        if overlapping_size / bbox_size > self.bbox_removal_threshold:\n            return normalize_bbox((0, 0, 0, 0), self.img_height, self.img_width)\n\n        return normalize_bbox(bbox, self.img_height, self.img_width)\n\n    def get_transform_init_args_names(self):\n        \"\"\"\n        Fetches the parameter(s) of __init__ method\n        :returns: tuple of parameter(s) of __init__ method\n        \"\"\"\n        return ('fill_value', 'bbox_removal_threshold', 'min_cutout_size', 'max_cutout_size', 'always_apply', 'p')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augmentation = albumentations.Compose([\n    CustomCutout(p=1),\n    albumentations.Flip(always_apply=True), # Either Horizontal, Vertical or both flips\n    albumentations.OneOf([  # One of blur or adding gauss noise\n        albumentations.Blur(p=0.50),  # Blurs the image\n        albumentations.GaussNoise(var_limit=5.0 / 255.0, p=0.50)  # Adds Gauss noise to image\n    ], p=1)\n], bbox_params = {\n    'format': 'pascal_voc',\n    'label_fields': ['labels']\n})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting a few images with and without augmentations","metadata":{}},{"cell_type":"code","source":"def get_bbox(bboxes, col, color='white'):\n    for i in range(len(bboxes)):\n        # Create a Rectangle patch\n        rect = patches.Rectangle(\n            (bboxes[i][0], bboxes[i][1]),\n            bboxes[i][2] - bboxes[i][0], \n            bboxes[i][3] - bboxes[i][1], \n            linewidth=2, \n            edgecolor=color, \n            facecolor='none')\n\n        # Add the patch to the Axes\n        col.add_patch(rect)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_images = 5\nrand_start = np.random.randint(0, len(image_ids) - 5)\nfig, ax = plt.subplots(nrows=num_images, ncols=2, figsize=(16, 40))\n\nfor index, image_id in enumerate(image_ids[rand_start : rand_start + num_images]):\n    # Read the image from image id\n    image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0  # Normalize\n    \n    # Get the bboxes details and apply all the augmentations\n    bboxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'x_max', 'y_max']].astype(np.int32).values\n    labels = np.ones((len(bboxes), ))  # As we have only one class (wheat heads)\n    aug_result = augmentation(image=image, bboxes=bboxes, labels=labels)\n\n    get_bbox(bboxes, ax[index][0], color='red')\n    ax[index][0].grid(False)\n    ax[index][0].set_xticks([])\n    ax[index][0].set_yticks([])\n    ax[index][0].title.set_text('Original Image')\n    ax[index][0].imshow(image)\n\n    get_bbox(aug_result['bboxes'], ax[index][1], color='red')\n    ax[index][1].grid(False)\n    ax[index][1].set_xticks([])\n    ax[index][1].set_yticks([])\n    ax[index][1].title.set_text(f'Augmented Image: Removed bboxes: {len(bboxes) - len(aug_result[\"bboxes\"])}')\n    ax[index][1].imshow(aug_result['image'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mixup augmentation\n\nIn mixup, two images are mixed with weights: $\\lambda$ and $1-\\lambda$. $\\lambda$ is generated from symmetric **beta** distribution with parameter **alpha**. This creates new virtual training samples.\n\nIn image classification images and labels can be mixed up as following:\n\n![Mixup Image](https://hoya012.github.io/assets/img/bag_of_trick/9.PNG)\n\nBut in object detection tasks, the labels are not one hot encoded classes and hence after mixing two images, the resultant image's label would be the union of bounding boxes of both the images and this makes implementation simpler.\n\nNow let's implement it.\n\nReferences: \n* Mixup original paper: https://arxiv.org/pdf/1710.09412.pdf\n* This kernel: https://www.kaggle.com/shonenkov/oof-evaluation-mixup-efficientdet\n* This kernel: https://www.kaggle.com/virajbagal/mixup-cutmix-fmix-visualisations","metadata":{}},{"cell_type":"code","source":"def mixup(images, bboxes, areas, alpha=1.0):\n    \"\"\"\n    Randomly mixes the given list if images with each other\n    \n    :param images: The images to be mixed up\n    :param bboxes: The bounding boxes (labels)\n    :param areas: The list of area of all the bboxes\n    :param alpha: Required to generate image wieghts (lambda) using beta distribution. In this case we'll use alpha=1, which is same as uniform distribution\n    \"\"\"\n    # Generate random indices to shuffle the images\n    indices = torch.randperm(len(images))\n    shuffled_images = images[indices]\n    shuffled_bboxes = bboxes[indices]\n    shuffled_areas = areas[indices]\n    \n    # Generate image weight (minimum 0.4 and maximum 0.6)\n    lam = np.clip(np.random.beta(alpha, alpha), 0.4, 0.6)\n    print(f'lambda: {lam}')\n    \n    # Weighted Mixup\n    mixedup_images = lam*images + (1 - lam)*shuffled_images\n    \n    mixedup_bboxes, mixedup_areas = [], []\n    for bbox, s_bbox, area, s_area in zip(bboxes, shuffled_bboxes, areas, shuffled_areas):\n        mixedup_bboxes.append(bbox + s_bbox)\n        mixedup_areas.append(area + s_area)\n    \n    return mixedup_images, mixedup_bboxes, mixedup_areas, indices.numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WheatDataset(Dataset):\n    \n    def __init__(self, df):\n        self.df = df\n        self.image_ids = self.df['image_id'].unique()\n\n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0  # Normalize\n        \n        # Get bbox coordinates for each wheat head(s)\n        bboxes_df = self.df[self.df['image_id'] == image_id]\n        boxes, areas = [], []\n        n_objects = len(bboxes_df)  # Number of wheat heads in the given image\n\n        for i in range(n_objects):\n            x_min = bboxes_df.iloc[i]['x_min']\n            x_max = bboxes_df.iloc[i]['x_max']\n            y_min = bboxes_df.iloc[i]['y_min']\n            y_max = bboxes_df.iloc[i]['y_max']\n\n            boxes.append([x_min, y_min, x_max, y_max])\n            areas.append(bboxes_df.iloc[i]['area'])\n\n        return {\n            'image_id': image_id,\n            'image': image,\n            'boxes': boxes,\n            'area': areas,\n        }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    images, bboxes, areas, image_ids = ([] for _ in range(4))\n    for data in batch:\n        images.append(data['image'])\n        bboxes.append(data['boxes'])\n        areas.append(data['area'])\n        image_ids.append(data['image_id'])\n\n    return np.array(images), np.array(bboxes), np.array(areas), np.array(image_ids)     ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = WheatDataset(train_df)\ntrain_loader = data_utils.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, collate_fn=collate_fn)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, bboxes, areas, image_ids = next(iter(train_loader))\naug_images, aug_bboxes, aug_areas, aug_indices = mixup(images, bboxes, areas)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_image(image_id):\n    \"\"\"Read the image from image id\"\"\"\n\n    image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0  # Normalize\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=5, ncols=3, figsize=(15, 30))\nfor index in range(5):\n    image_id = image_ids[index]\n    image = read_image(image_id)\n\n    get_bbox(bboxes[index], ax[index][0], color='red')\n    ax[index][0].grid(False)\n    ax[index][0].set_xticks([])\n    ax[index][0].set_yticks([])\n    ax[index][0].title.set_text('Original Image #1')\n    ax[index][0].imshow(image)\n    \n    image_id = image_ids[aug_indices[index]]\n    image = read_image(image_id)\n    get_bbox(bboxes[aug_indices[index]], ax[index][1], color='red')\n    ax[index][1].grid(False)\n    ax[index][1].set_xticks([])\n    ax[index][1].set_yticks([])\n    ax[index][1].title.set_text('Original Image #2')\n    ax[index][1].imshow(image)\n\n    get_bbox(aug_bboxes[index], ax[index][2], color='red')\n    ax[index][2].grid(False)\n    ax[index][2].set_xticks([])\n    ax[index][2].set_yticks([])\n    ax[index][2].title.set_text(f'Augmented Image: lambda * image1 + (1 - lambda) * image2')\n    ax[index][2].imshow(aug_images[index])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now let's apply this augmentations to entire dataset and create a larger dataset\n\nNote that applying augmentations on the fly is better than creating new dataset (in almost all the cases). But for initial iteration, I'll still save the images and will create a new dataset as applying Mixup on the fly might be tricky. The reason is as following:\n\nWhen you create a larger dataset using any augmentations, you randomly apply those augmentations on the images and save them. Now when you train on those images, your model gets to see only those augmentations at each epoch (no new augmentations are randomly generated at every epoch in this process and thus this limits the number of possibilities for each image).\n\nBut when you augment the data on the fly, each image can have different augmentation applied at each epoch (because of randomness) but this is not the case with the previous approach. And that's why I think the second approach works well.","metadata":{}},{"cell_type":"markdown","source":"## Below code uses all above augmentations except Mixup. Refer next section for only Mixup","metadata":{}},{"cell_type":"code","source":"augmentation = albumentations.Compose([\n    albumentations.Flip(p=0.60),\n    albumentations.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.60),\n    albumentations.HueSaturationValue(p=0.60)\n], bbox_params = {\n    'format': 'pascal_voc',\n    'label_fields': ['labels']\n})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isdir('train'):\n    os.mkdir('train')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_dataset(index, image_id):\n    # Read the image from image id\n    image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    # Get the bboxes details and apply all the augmentations\n    bboxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'x_max', 'y_max']].astype(np.int32).values\n    source = train_df[train_df['image_id'] == image_id]['source'].unique()[0]\n    labels = np.ones((len(bboxes), ))  # As we have only one class (wheat heads)\n    aug_result = augmentation(image=image, bboxes=bboxes, labels=labels)\n\n    aug_image = aug_result['image']\n    aug_bboxes = aug_result['bboxes']\n    \n    Image.fromarray(image).save(os.path.join(WORK_DIR, 'train', f'{image_id}.jpg'))\n    Image.fromarray(aug_image).save(os.path.join(WORK_DIR, 'train', f'{image_id}_aug.jpg'))\n\n    image_metadata = []\n    for bbox in aug_bboxes:\n        bbox = tuple(map(int, bbox))\n        image_metadata.append({\n            'image_id': f'{image_id}_aug',\n            'x_min': bbox[0],\n            'y_min': bbox[1],\n            'x_max': bbox[2],\n            'y_max': bbox[3],\n            'width': bbox[2] - bbox[0],\n            'height': bbox[3] - bbox[1],\n            'area': (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]),\n            'source': source\n        })\n    return image_metadata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_metadata = Parallel(n_jobs=8)(delayed(create_dataset)(index, image_id) for index, image_id in tqdm(enumerate(image_ids), total=len(image_ids)))\nimage_metadata = [item for sublist in image_metadata for item in sublist]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aug_train_df = pd.DataFrame(image_metadata)\naug_train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(aug_train_df.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([train_df, aug_train_df]).reset_index(drop=True)\ntrain_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now let's split the entire dataset into 5 folds with stratification on source.\n\nReference: https://www.kaggle.com/alexandersoare/how-to-prepare-a-stratified-split","metadata":{}},{"cell_type":"code","source":"# Add a new column to store kfold indices\ntrain_df.loc[:, 'kfold'] = -1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_source = train_df[['image_id', 'source']].drop_duplicates()\n\n# get lists for image_ids and sources\nimage_ids = image_source['image_id'].to_numpy()\nsources = image_source['source'].to_numpy()\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1996)\nsplit = skf.split(image_ids, sources) # second arguement is what we are stratifying by\n\nfor fold, (train_idx, val_idx) in enumerate(split):\n    translated_val_idx = train_df[train_df['image_id'].isin(image_ids[val_idx])].index.values\n    print(len(translated_val_idx))\n    train_df.loc[translated_val_idx, 'kfold'] = fold","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('train.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r -qq train.zip train\n!rm -rf train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's create new dataset using only mixup augmentation (TODO)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}