{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport ast\nfrom collections import namedtuple\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport joblib\nfrom joblib import Parallel, delayed\n\nimport cv2\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations.core.transforms_interface import DualTransform\nfrom albumentations.augmentations.bbox_utils import denormalize_bbox, normalize_bbox\n\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.utils.data as data_utils\n\nfrom matplotlib import pyplot as plt\nimport matplotlib.patches as patches\nfrom matplotlib.image import imsave","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_DIR = '/kaggle/input/global-wheat-detection'\nWORK_DIR = '/kaggle/working'\nBATCH_SIZE = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(1996)\ntrain_df = pd.read_csv(os.path.join(BASE_DIR)+'/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[['x_min','y_min', 'width', 'height']] = pd.DataFrame([ast.literal_eval(x) for x in train_df.bbox.tolist()], index= train_df.index)\ntrain_df = train_df[['image_id', 'bbox', 'source', 'x_min', 'y_min', 'width', 'height']]\ntrain_df['area'] = train_df['width'] * train_df['height']\ntrain_df['x_max'] = train_df['x_min'] + train_df['width']\ntrain_df['y_max'] = train_df['y_min'] + train_df['height']\ntrain_df = train_df.drop(['bbox'], axis=1)\ntrain_df = train_df[['image_id', 'x_min', 'y_min', 'x_max', 'y_max', 'width', 'height', 'area', 'source']]\n\n# remove the faulty bounding boxes\ntrain_df = train_df[train_df['area'] < 100000]\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_ids = train_df[\"image_id\"].nunique()\nimage_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"b6ab77fd7\"\nimg = cv2.imread(os.path.join(BASE_DIR)+'/train'+f'/{image_id}.jpg',cv2.IMREAD_COLOR)\nimg = cv2.cvtColor(img,cv2.COLOR_BGR2RGB).astype(np.float32)\nimg/=255.0\nplt.figure(figsize = (5,5))\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pascal_voc_boxes = train_df[train_df[\"image_id\"]==image_id][['x_min','y_min','x_max','y_max']].astype(np.float32).values\n# pascal_voc_boxes.shape\ncoco_box = coco_boxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'width', 'height']].astype(np.int32).values\ncoco_box.shape\nassert(len(pascal_voc_boxes)==len(coco_box))\nlabels = np.ones(len(pascal_voc_boxes),)\nlabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bbox(bboxes,col,color='white',bbox_format = 'pascal_voc'):\n    for i in range(len(bboxes)):\n        if bbox_format == 'pascal_voc':\n            rect = patches.Rectangle(\n                (bboxes[i][0], bboxes[i][1]),\n                bboxes[i][2] - bboxes[i][0], \n                bboxes[i][3] - bboxes[i][1], \n                linewidth=2, \n                edgecolor=color, \n                facecolor='none')\n            \n        else:\n            rect = patches.Rectangle(\n                (bboxes[i][0], bboxes[i][1]),\n                bboxes[i][2], \n                bboxes[i][3], \n                linewidth=2, \n                edgecolor=color, \n                facecolor='none')\n        col.add_patch(rect)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = albumentations.Compose([\n        albumentations.Resize(512, 512),   # Resize the given 1024 x 1024 image to 512 * 512\n        albumentations.VerticalFlip(1),    # Verticlly flip the image\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_result = aug(image=img, bboxes=pascal_voc_boxes, labels=labels)\n# aug_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(nrows =1 ,ncols = 2,figsize = (10,10))\nget_bbox(pascal_voc_boxes,ax[0],color = 'red')\nax[0].title.set_text(\"Original Image\")\nax[0].imshow(img)\n\nget_bbox(aug_result['bboxes'], ax[1], color='red')\nax[1].title.set_text('Augmented Image')\nax[1].imshow(aug_result['image'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\naug = albumentations.Compose([\n        albumentations.Resize(512, 512),   # Resize the given 1024 x 1024 image to 512 * 512\n        albumentations.VerticalFlip(1),     # Verticlly flip the image\n        albumentations.Blur(p=1)\n    ], bbox_params={'format': 'coco', 'label_fields': ['labels']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_result = aug(image=img, bboxes=coco_boxes, labels=labels)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(10, 10))\nget_bbox(coco_boxes, ax[0], color='red', bbox_format='coco')\nax[0].title.set_text('Original Image')\nax[0].imshow(img)\n\nget_bbox(aug_result['bboxes'], ax[1], color='red', bbox_format='coco')\nax[1].title.set_text('Augmented Image')\nax[1].imshow(aug_result['image'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cutout Implementation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass CustomCutout(DualTransform):\n    def __init__(\n        self,\n        fill_value=0,\n        bbox_removal_threshold=0.50,\n        min_cutout_size=192,\n        max_cutout_size=512,\n        always_apply=False,\n        p=0.5\n    ):\n        \"\"\"\n        Class constructor\n        \n        :param fill_value: Value to be filled in cutout (default is 0 or black color)\n        :param bbox_removal_threshold: Bboxes having content cut by cutout path more than this threshold will be removed\n        :param min_cutout_size: minimum size of cutout (192 x 192)\n        :param max_cutout_size: maximum size of cutout (512 x 512)\n        \"\"\"\n        super(CustomCutout, self).__init__(always_apply, p)  # Initialize parent class\n        self.fill_value = fill_value\n        self.bbox_removal_threshold = bbox_removal_threshold\n        self.min_cutout_size = min_cutout_size\n        self.max_cutout_size = max_cutout_size\n        \n    def _get_cutout_position(self, img_height, img_width, cutout_size):\n        \"\"\"\n        Randomly generates cutout position as a named tuple\n        \n        :param img_height: height of the original image\n        :param img_width: width of the original image\n        :param cutout_size: size of the cutout patch (square)\n        :returns position of cutout patch as a named tuple\n        \"\"\"\n        position = namedtuple('Point', 'x y')\n        return position(\n            np.random.randint(0, img_width - cutout_size + 1),\n            np.random.randint(0, img_height - cutout_size + 1)\n        )\n        \n    def _get_cutout(self, img_height, img_width):\n        \"\"\"\n        Creates a cutout pacth with given fill value and determines the position in the original image\n        \n        :param img_height: height of the original image\n        :param img_width: width of the original image\n        :returns (cutout patch, cutout size, cutout position)\n        \"\"\"\n        cutout_size = np.random.randint(self.min_cutout_size, self.max_cutout_size + 1)\n        cutout_position = self._get_cutout_position(img_height, img_width, cutout_size)\n        return np.full((cutout_size, cutout_size, 3), self.fill_value), cutout_size, cutout_position\n        \n    def apply(self, image, **params):\n        \"\"\"\n        Applies the cutout augmentation on the given image\n        \n        :param image: The image to be augmented\n        :returns augmented image\n        \"\"\"\n        image = image.copy()  # Don't change the original image\n        self.img_height, self.img_width, _ = image.shape\n        cutout_arr, cutout_size, cutout_pos = self._get_cutout(self.img_height, self.img_width)\n        \n        # Set to instance variables to use this later\n        self.image = image\n        self.cutout_pos = cutout_pos\n        self.cutout_size = cutout_size\n        \n        image[cutout_pos.y:cutout_pos.y+cutout_size, cutout_pos.x:cutout_size+cutout_pos.x, :] = cutout_arr\n        return image\n    \n    def apply_to_bbox(self, bbox, **params):\n        \"\"\"\n        Removes the bounding boxes which are covered by the applied cutout\n        \n        :param bbox: A single bounding box coordinates in pascal_voc format\n        :returns transformed bbox's coordinates\n        \"\"\"\n\n        # Denormalize the bbox coordinates\n        bbox = denormalize_bbox(bbox, self.img_height, self.img_width)\n        x_min, y_min, x_max, y_max = tuple(map(int, bbox))\n\n        bbox_size = (x_max - x_min) * (y_max - y_min)  # width * height\n        overlapping_size = np.sum(\n            (self.image[y_min:y_max, x_min:x_max, 0] == self.fill_value) &\n            (self.image[y_min:y_max, x_min:x_max, 1] == self.fill_value) &\n            (self.image[y_min:y_max, x_min:x_max, 2] == self.fill_value)\n        )\n\n        # Remove the bbox if it has more than some threshold of content is inside the cutout patch\n        if overlapping_size / bbox_size > self.bbox_removal_threshold:\n            return normalize_bbox((0, 0, 0, 0), self.img_height, self.img_width)\n\n        return normalize_bbox(bbox, self.img_height, self.img_width)\n\n    def get_transform_init_args_names(self):\n        \"\"\"\n        Fetches the parameter(s) of __init__ method\n        :returns: tuple of parameter(s) of __init__ method\n        \"\"\"\n        return ('fill_value', 'bbox_removal_threshold', 'min_cutout_size', 'max_cutout_size', 'always_apply', 'p')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentation = albumentations.Compose([\n    CustomCutout(p=1),\n    albumentations.Flip(always_apply=True), # Either Horizontal, Vertical or both flips\n    albumentations.OneOf([  # One of blur or adding gauss noise\n        albumentations.Blur(p=0.50),  # Blurs the image\n        albumentations.GaussNoise(var_limit=5.0 / 255.0, p=0.50)  # Adds Gauss noise to image\n    ], p=1)\n], bbox_params = {\n    'format': 'pascal_voc',\n    'label_fields': ['labels']\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bbox(bboxes, col, color='white'):\n    for i in range(len(bboxes)):\n        # Create a Rectangle patch\n        rect = patches.Rectangle(\n            (bboxes[i][0], bboxes[i][1]),\n            bboxes[i][2] - bboxes[i][0], \n            bboxes[i][3] - bboxes[i][1], \n            linewidth=2, \n            edgecolor=color, \n            facecolor='none')\n\n        # Add the patch to the Axes\n        col.add_patch(rect)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_images = 5\nrand_start = np.random.randint(0, len(train_df['image_id']) - 5)\nfig, ax = plt.subplots(nrows=num_images, ncols=2, figsize=(16, 20))\n\nfor index, image_id in enumerate(train_df['image_id'][rand_start : rand_start + num_images]):\n    # Read the image from image id\n    image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0  # Normalize\n    \n    # Get the bboxes details and apply all the augmentations\n    bboxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'x_max', 'y_max']].astype(np.int32).values\n    labels = np.ones((len(bboxes), ))  # As we have only one class (wheat heads)\n    aug_result = augmentation(image=image, bboxes=bboxes, labels=labels)\n\n    get_bbox(bboxes, ax[index][0], color='red')\n    ax[index][0].grid(False)\n    ax[index][0].set_xticks([])\n    ax[index][0].set_yticks([])\n    ax[index][0].title.set_text('Original Image')\n    ax[index][0].imshow(image)\n\n    get_bbox(aug_result['bboxes'], ax[index][1], color='red')\n    ax[index][1].grid(False)\n    ax[index][1].set_xticks([])\n    ax[index][1].set_yticks([])\n    ax[index][1].title.set_text(f'Augmented Image: Removed bboxes: {len(bboxes) - len(aug_result[\"bboxes\"])}')\n    ax[index][1].imshow(aug_result['image'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mixup Implementation ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef mixup(images, bboxes, areas, alpha=1.0):\n    \"\"\"\n    Randomly mixes the given list if images with each other\n    \n    :param images: The images to be mixed up\n    :param bboxes: The bounding boxes (labels)\n    :param areas: The list of area of all the bboxes\n    :param alpha: Required to generate image wieghts (lambda) using beta distribution. In this case we'll use alpha=1, which is same as uniform distribution\n    \"\"\"\n    # Generate random indices to shuffle the images\n    indices = torch.randperm(len(images))\n    shuffled_images = images[indices]\n    shuffled_bboxes = bboxes[indices]\n    shuffled_areas = areas[indices]\n    \n    # Generate image weight (minimum 0.4 and maximum 0.6)\n    lam = np.clip(np.random.beta(alpha, alpha), 0.4, 0.6)\n    print(f'lambda: {lam}')\n    \n    # Weighted Mixup\n    mixedup_images = lam*images + (1 - lam)*shuffled_images\n    \n    mixedup_bboxes, mixedup_areas = [], []\n    for bbox, s_bbox, area, s_area in zip(bboxes, shuffled_bboxes, areas, shuffled_areas):\n        mixedup_bboxes.append(bbox + s_bbox)\n        mixedup_areas.append(area + s_area)\n    \n    return mixedup_images, mixedup_bboxes, mixedup_areas, indices.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass WheatDataset(Dataset):\n    \n    def __init__(self, df):\n        self.df = df\n        self.image_ids = self.df['image_id'].unique()\n\n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0  # Normalize\n        \n        # Get bbox coordinates for each wheat head(s)\n        bboxes_df = self.df[self.df['image_id'] == image_id]\n        boxes, areas = [], []\n        n_objects = len(bboxes_df)  # Number of wheat heads in the given image\n\n        for i in range(n_objects):\n            x_min = bboxes_df.iloc[i]['x_min']\n            x_max = bboxes_df.iloc[i]['x_max']\n            y_min = bboxes_df.iloc[i]['y_min']\n            y_max = bboxes_df.iloc[i]['y_max']\n\n            boxes.append([x_min, y_min, x_max, y_max])\n            areas.append(bboxes_df.iloc[i]['area'])\n\n        return {\n            'image_id': image_id,\n            'image': image,\n            'boxes': boxes,\n            'area': areas,\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef collate_fn(batch):\n    images, bboxes, areas, image_ids = ([] for _ in range(4))\n    for data in batch:\n        images.append(data['image'])\n        bboxes.append(data['boxes'])\n        areas.append(data['area'])\n        image_ids.append(data['image_id'])\n\n    return np.array(images), np.array(bboxes), np.array(areas), np.array(image_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = WheatDataset(train_df)\n# train_dataset.__getitem__(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = data_utils.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, collate_fn=collate_fn)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, bboxes, areas, image_ids = next(iter(train_loader))\naug_images, aug_bboxes, aug_areas, aug_indices = mixup(images, bboxes, areas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(image_id):\n    \"\"\"Read the image from image id\"\"\"\n\n    image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0  # Normalize\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=5, ncols=3, figsize=(15, 20))\nfor index in range(5):\n    image_id = image_ids[index]\n    image = read_image(image_id)\n\n    get_bbox(bboxes[index], ax[index][0], color='red')\n    ax[index][0].grid(False)\n    ax[index][0].set_xticks([])\n    ax[index][0].set_yticks([])\n    ax[index][0].title.set_text('Original Image #1')\n    ax[index][0].imshow(image)\n    \n    image_id = image_ids[aug_indices[index]]\n    image = read_image(image_id)\n    get_bbox(bboxes[aug_indices[index]], ax[index][1], color='red')\n    ax[index][1].grid(False)\n    ax[index][1].set_xticks([])\n    ax[index][1].set_yticks([])\n    ax[index][1].title.set_text('Original Image #2')\n    ax[index][1].imshow(image)\n\n    get_bbox(aug_bboxes[index], ax[index][2], color='red')\n    ax[index][2].grid(False)\n    ax[index][2].set_xticks([])\n    ax[index][2].set_yticks([])\n    ax[index][2].title.set_text(f'Augmented Image: lambda * image1 + (1 - lambda) * image2')\n    ax[index][2].imshow(aug_images[index])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentation = albumentations.Compose([\n    albumentations.Flip(p=0.60),\n    albumentations.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.60),\n    albumentations.HueSaturationValue(p=0.60)\n], bbox_params = {\n    'format': 'pascal_voc',\n    'label_fields': ['labels']\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    \ndef create_dataset(index, image_id):\n    # Read the image from image id\n    image = cv2.imread(os.path.join(BASE_DIR, 'train', f'{image_id}.jpg'), cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    # Get the bboxes details and apply all the augmentations\n    bboxes = train_df[train_df['image_id'] == image_id][['x_min', 'y_min', 'x_max', 'y_max']].astype(np.int32).values\n    source = train_df[train_df['image_id'] == image_id]['source'].unique()[0]\n    labels = np.ones((len(bboxes), ))  # As we have only one class (wheat heads)\n    aug_result = augmentation(image=image, bboxes=bboxes, labels=labels)\n\n    aug_image = aug_result['image']\n    aug_bboxes = aug_result['bboxes']\n    \n    Image.fromarray(image).save(os.path.join(WORK_DIR, 'train', f'{image_id}.jpg'))\n    Image.fromarray(aug_image).save(os.path.join(WORK_DIR, 'train', f'{image_id}_aug.jpg'))\n\n    image_metadata = []\n    for bbox in aug_bboxes:\n        bbox = tuple(map(int, bbox))\n        image_metadata.append({\n            'image_id': f'{image_id}_aug',\n            'x_min': bbox[0],\n            'y_min': bbox[1],\n            'x_max': bbox[2],\n            'y_max': bbox[3],\n            'width': bbox[2] - bbox[0],\n            'height': bbox[3] - bbox[1],\n            'area': (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]),\n            'source': source\n        })\n    return image_metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nif not os.path.isdir('train'):\n    os.mkdir('train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_metadata = Parallel(n_jobs=8)(delayed(create_dataset)(index, image_id) for index, image_id in tqdm(enumerate(image_ids), total=len(image_ids)))\nimage_metadata = [item for sublist in image_metadata for item in sublist]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_images = pd.DataFrame(image_metadata)\naug_images.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final =  pd.concat([train_df,aug_images],axis=0).reset_index(drop = True)\nfinal","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Folds of Dataset(final)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_source = final[['image_id', 'source']].drop_duplicates()\nimage_source","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_list = image_source[\"image_id\"].to_numpy()\nsources = image_source['source'].to_numpy()\n# image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom sklearn.model_selection import StratifiedKFold\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom itertools import islice\n\nskf = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)\nsplit = skf.split(image_list, sources)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"select = 0\ntrain_ix, val_ix = next(islice(split, select, select+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = image_list[train_ix]\nval_ids = image_list[val_ix]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = final[final['image_id'].isin(train_ids)]\nval_df = final[final['image_id'].isin(val_ids)]\n# train_df\nval_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'# train images: {train_ids.shape[0]}')\nprint(f'# val images: {val_ids.shape[0]}')\n\nfig = plt.figure(figsize=(20, 5))\ncounts = train_df['source'].value_counts()\nax1 = fig.add_subplot(1,2,1)\na = ax1.bar(counts.index, counts)\ncounts = val_df['source'].value_counts()\nax2 = fig.add_subplot(1,2,2)\na = ax2.bar(counts.index, counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}