{"cells":[{"metadata":{},"cell_type":"markdown","source":"# YOLOv3 Detector + Submission","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"* This notebook is based on [this](http://www.kaggle.com/mattbast/object-detection-tensorflow-end-to-end) great notebook. I've just added the Inference and the submission code, and \"massaged\" the original a bit so that it is more intuitive, I hope (at leas for me it is).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 1 - Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nfrom numpy.random import (\n    choice\n)\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import (\n    Rectangle\n)\n\nimport tensorflow as tf\nfrom tensorflow.keras import (\n    Input, \n    Model\n)\nfrom tensorflow.keras.layers import (\n    Conv2D,\n    BatchNormalization,\n    LeakyReLU,\n    Add  \n)\nfrom tensorflow.keras.optimizers import (\n    Adam\n)\nfrom tensorflow.keras.callbacks import (\n    ReduceLROnPlateau,\n    EarlyStopping,\n    TensorBoard\n)\nfrom tensorflow.keras.losses import (\n    BinaryCrossentropy,\n    Reduction\n)\nfrom PIL import Image, ImageDraw, ImageEnhance\n\nimport albumentations as A\n\nfrom tqdm import tqdm\n\nfrom pathlib import Path\n\nfrom importlib import reload as reload_lib\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2 - Paths","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_root_dir = Path('../input/global-wheat-detection')\ndata_root_dir.is_dir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv_file = data_root_dir / 'train.csv'\ntrain_csv_file.is_file()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3 - Data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 3.1 - Load the data data frame","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df = pd.read_csv(train_csv_file)\ndata_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.2 - Extract the bboxes","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 3.2.1 - Auxiliary functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_bbox_from_str(df_line):\n    bbox = df_line['bbox'].str.split(',', expand=True)\n    bbox[0] = bbox[0].str.slice(start=1)\n    bbox[3] = bbox[3].str.slice(stop=-1)\n    \n    return bbox.values.astype(float)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.2.2 - Main","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df = data_df.groupby('image_id').apply(extract_bbox_from_str)\ndata_df['b6ab77fd7'][0:5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.3 - Train / val split","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"N = data_df.shape[0]  # total number of samples\ntest_n = 10  # number of samples for final test\n\ntrain_image_ids = np.unique(data_df.index.values)[:N-test_n]\nval_image_ids = np.unique(data_df.index.values)[N-test_n:]\nprint(f'Number of train images: {train_image_ids.shape[0]}\\nNumber of validation images: {val_image_ids.shape[0]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.4 - Load the images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 3.4.1 - Auxiliary functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_images(data_df, data_root_dir, image_ids, data_type, resize_shape=None):\n    def _load_image(img_root_dir, img_id, resize_shape):\n        return np.asarray(Image.open(str(img_root_dir / (img_id+'.jpg'))).resize(resize_shape))\n\n    images_dict = {}\n    bboxes_dict = {}\n\n    for img_id in tqdm(image_ids):\n        images_dict[img_id] = _load_image(img_root_dir=data_root_dir / data_type, img_id=img_id, resize_shape=(256, 256))\n        bboxes_dict[img_id] = data_df[img_id].copy() / 4\n        \n    return images_dict, bboxes_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.4.2 - Main","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"resize_shape = (256, 256)\n\ntrain_images_dict, train_bboxes_dict = load_images(data_df=data_df, data_root_dir=data_root_dir, image_ids=train_image_ids, data_type='train', resize_shape=resize_shape)\nval_images_dict, val_bboxes_dict = load_images(data_df=data_df, data_root_dir=data_root_dir, image_ids=val_image_ids, data_type='train', resize_shape=resize_shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.5 - Visualization","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 3.5.1 - Auxiliary functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image_sample(images, bboxes, sample_size=5):\n    def _image_bbox_viz(ax, img, bboxes):\n        ax.imshow(img)\n        \n        for bbox in bboxes:\n            x, y, w, h = bbox\n            ax.add_patch(Rectangle((x, y), w, h, fill=False, lw=1.5, color='red'))\n            \n        return np.asarray(ax)\n    fig, axs = plt.subplots(1, sample_size, figsize=(20, 20))\n    if sample_size > 1:\n        for idx, img in enumerate(images):\n            _image_bbox_viz(axs[idx], img, bboxes[idx])\n    else:\n        _image_bbox_viz(axs, images[0], bboxes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.5.2 - Main","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"N = len(train_images_dict.values())\nsample_size = 6\n\nrand_sample_idx = choice(N, sample_size)\nsample_images = np.array(list(train_images_dict.values()))[rand_sample_idx]\nsample_bboxes = np.array(list(train_bboxes_dict.values()))[rand_sample_idx]\n\nshow_image_sample(\n    images=sample_images, \n    bboxes=sample_bboxes, \n    sample_size=sample_size\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.6 - Cleaning bboxes","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 3.6.1 - Auxiliary functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_bboxes(images_dict, bboxes_dict, min_bbox_area, max_bbox_area, clean=False, excluded_bboxes=None):\n    small_bbox_area_cnt, large_bbox_area_cnt = (0, 0)\n    for img_id in tqdm(bboxes_dict):\n        bboxes = bboxes_dict[img_id]\n        delete_bbox_idx = []\n        for bbox_idx, bbox in enumerate(bboxes):\n            if excluded_bboxes is not None:\n                if (img_id, bbox_idx) in excluded_bboxes:\n                    continue\n            _, _, w, h = bbox\n            if w * h <= min_bbox_area or w * h >= max_bbox_area:\n                if w * h >= max_bbox_area:\n                    # print(f'w * h = {w * h}')\n                    print(f'image id: {img_id}, bbox index: {bbox_idx}')\n                    show_image_sample(\n                        images=[images_dict[img_id]], \n                        bboxes=[bbox], \n                        sample_size=1\n                    )\n                    large_bbox_area_cnt += 1\n                else:\n                    # print(f'w * h = {w * h}')\n                    small_bbox_area_cnt += 1\n                delete_bbox_idx.append(bbox_idx)\n        if clean:\n            bboxes_dict[img_id] = np.delete(bboxes, delete_bbox_idx, axis=0)\n                \n    print(f'Small area bboxes: {small_bbox_area_cnt}\\nLarge area bboxes: {large_bbox_area_cnt}')\n    return bboxes_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.6.2 - Main","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_train_bboxes_dict = clean_bboxes(\n    images_dict=train_images_dict,\n    bboxes_dict=train_bboxes_dict.copy(), \n    min_bbox_area=10, \n    max_bbox_area=8000,\n    clean=True,\n    excluded_bboxes=[('51f2e0a05', 5), ('69fc3d3ff', 1)]\n)\n\n# check\n# image id: 51f2e0a05, bbox index: 5\nimage_id = '51f2e0a05'\nbbox_idx = 5\nshow_image_sample(\n    images=[train_images_dict[image_id]], \n    bboxes=[train_bboxes_dict[image_id][bbox_idx]], \n    sample_size=1\n) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.7 - Classes","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 3.7.1 - Data Generator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, image_ids, image_pixels, labels=None, batch_size=1, shuffle=False, augment=False):\n        self.image_ids = image_ids\n        self.image_pixels = image_pixels\n        self.labels = labels\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        \n        self.on_epoch_end()\n        \n        self.image_grid = self.form_image_grid()\n            \n    def on_epoch_end(self):\n        self.indexes  = np.arange(len(self.image_ids))\n        \n        if self.shuffle:\n            np.random.shuffle(self.indexes)\n            \n    def form_image_grid(self):\n        image_grid = np.zeros((32, 32, 4))\n        \n        # initial cell coordinates\n        cell = [0, 0, 256 / 32, 256 / 32]\n            \n        for i in range(32):\n            for j in range(32):\n                image_grid[i, j] = cell\n                cell[0] = cell[0] + cell[2]\n            cell[0] = 0\n            cell[1] = cell[1] + cell[3]\n\n        return image_grid\n    \n    def __len__(self):\n        return int(np.floor(len(self.image_ids) / self.batch_size))\n    \n    def __getitem__(self, index):\n        indexes = self.indexes[index * self.batch_size : (index + 1) * self.batch_size]\n        \n        batch_image_ids = [self.image_ids[i] for i in indexes]\n\n        return self.__get_batch(batch_image_ids)\n    \n    def __get_batch(self, batch_image_ids):\n        X, y = [], []\n        \n        for idx, image_id in enumerate(batch_image_ids):\n            batch_image_pixels = self.image_pixels[image_id]\n            batch_image_bboxes = self.labels[image_id]\n            \n            if self.augment:\n                batch_image_pixels, batch_image_bboxes = self.augment_image(batch_image_pixels, batch_image_bboxes)\n            else:\n                batch_image_pixels = self.contrast_image(batch_image_pixels)\n                batch_image_bboxes = self.form_label_grid(batch_image_bboxes)\n            X.append(batch_image_pixels)\n            y.append(batch_image_bboxes)\n        return np.array(X), np.array(y)\n    \n    def augment_image(self, image_pixels, image_bboxes):\n        bbox_labels = np.ones(len(image_bboxes))\n        \n        aug_result = self.train_aug(image=image_pixels, bboxes=image_bboxes, labels=bbox_labels)\n        \n        image_bboxes = self.form_label_grid(aug_result['bboxes'])\n        \n        return np.array(aug_result['image']) / 255, image_bboxes\n    \n    def contrast_image(self, image_pixels):\n        aug_result = self.val_aug(image=image_pixels)\n        return np.array(aug_result['image']) / 255\n    \n    \n    def form_label_grid(self, bboxes):\n        label_grid = np.zeros((32, 32, 10))\n        \n        for i in range(32):\n            for j in range(32):\n                cell = self.image_grid[i, j]\n                label_grid[i, j] = self.rect_intersect(cell, bboxes)\n        return label_grid\n    \n    def rect_intersect(self, cell, bboxes):\n        cell_x, cell_y, cell_width, cell_height = cell\n        cell_x_max = cell_x + cell_width\n        cell_y_max = cell_y + cell_height\n        \n        anchor_one = np.array([0, 0, 0, 0, 0])\n        anchor_two = np.array([0, 0, 0, 0, 0])\n        \n        for bbox in bboxes:\n            bbox_x, bbox_y, bbox_width, bbox_height = bbox\n            bbox_center_x = bbox_x + (bbox_width / 2)\n            bbox_center_y = bbox_y + (bbox_height / 2)\n            \n            if bbox_center_x >= cell_x and bbox_center_x < cell_x_max and bbox_center_y >= cell_y and bbox_center_y < cell_y_max:\n                if anchor_one[0] == 0:  # if there is no object present in the anchor 1 cell\n                    anchor_one = self.yolo_shape(\n                        [bbox_x, bbox_y, bbox_width, bbox_height],\n                        [cell_x, cell_y, cell_width, cell_height]\n                    )\n                elif anchor_two[0] == 0:  # if there is no object present in the anchor 2 cell\n                    anchor_two = self.yolo_shape(\n                        [bbox_x, bbox_y, bbox_width, bbox_height],\n                        [cell_x, cell_y, cell_width, cell_height]\n                    )\n                else:\n                    break\n        return np.concatenate((anchor_one, anchor_two), axis=None)\n    \n    def yolo_shape(self, bbox, cell):\n        bbox_x, bbox_y, bbox_width, bbox_height = bbox\n        cell_x, cell_y, cell_width, cell_height = cell\n        \n        # move the top left x, y coordinates to the center\n        bbox_x = bbox_x + (bbox_width / 2)\n        bbox_y = bbox_y + (bbox_height / 2)\n        \n        # x, y relative to cell\n        bbox_x = (bbox_x - cell_x) / cell_width\n        bbox_y = (bbox_y - cell_y) / cell_height\n        \n        # change the bbox width and height relative to the cell width and height\n        bbox_width = bbox_width / 256\n        bbox_height = bbox_height / 256\n        \n        return [1, bbox_x, bbox_y, bbox_width, bbox_height]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.7.2 - Data Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DataGenerator.train_aug = A.Compose([\n        A.RandomSizedCrop(\n            min_max_height=(200, 200), \n            height=256, \n            width=256, \n            p=0.8\n        ),\n        A.OneOf([\n            A.Flip(),\n            A.RandomRotate90(),\n        ], p=1),\n        A.OneOf([\n            A.HueSaturationValue(),\n            A.RandomBrightnessContrast()\n        ], p=1),\n        A.OneOf([\n            A.GaussNoise(),\n            A.GlassBlur(),\n            A.ISONoise(),\n            A.MultiplicativeNoise(),\n        ], p=0.5),\n        A.Cutout(\n            num_holes=8, \n            max_h_size=16, \n            max_w_size=16, \n            fill_value=0, \n            p=0.5\n        ),\n        A.CLAHE(p=1),\n        A.ToGray(p=1),\n    ], \n    bbox_params={'format': 'coco', 'label_fields': ['labels']})\n    \nDataGenerator.val_aug = A.Compose([\n    A.CLAHE(p=1),\n    A.ToGray(p=1),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.8 DataGenerators creation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = DataGenerator(\n    train_image_ids,\n    train_images_dict,\n    train_bboxes_dict,\n    batch_size=6,\n    shuffle=True,\n    augment=True\n)\nimage_grid = train_generator.image_grid\n\nval_generator = DataGenerator(\n    val_image_ids,\n    val_images_dict,\n    val_bboxes_dict,\n    batch_size=10,\n    shuffle=False, \n    augment=False\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4 - Model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 4.1 - Classes","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 4.1.1 - YOLOv3","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class YOLOv3:\n    def __init__(self):\n        self.darknet_53 = None\n        self.build_net()\n\n    def build_net(self):\n        # == INPUT ==   \n        print(f'Working on:\\n\\t>Input layers')\n        X_input = Input(shape=(256, 256, 3))\n\n        X = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(X_input)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        # == BLOCK 1 ==\n        print(f'Working on:\\n\\t>Block 1')\n\n        X = Conv2D(64, (3, 3), strides=(1, 1), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X_sc = X\n\n        for layer_idx in tqdm(range(2)):\n            X = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Conv2D(64, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Add()([X_sc, X])\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X_sc = X\n            \n        # == BLOCK 2 ==\n        print(f'Working on:\\n\\t>Block 2')\n\n        X = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X_sc = X\n\n        for layer_idx in tqdm(range(2)):\n            X = Conv2D(64, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = tf.keras.layers.Add()([X_sc, X])\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X_sc = X\n            \n        # == BLOCK 3 ==\n        print(f'Working on:\\n\\t>Block 3')\n        X = Conv2D(256, (3, 3), strides=(2, 2), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X_sc = X\n\n        for layer_idx in tqdm(range(8)):\n            X = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Conv2D(256, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Add()([X_sc, X])\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X_sc = X\n            \n        # == BLOCK 4 ==\n        print(f'Working on:\\n\\t>Block 4')\n        X = Conv2D(512, (3, 3), strides=(2, 2), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X_sc = X\n\n        for layer_idx in tqdm(range(8)):\n            X = Conv2D(256, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Conv2D(512, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Add()([X_sc, X])\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X_sc = X\n            \n        # == BLOCK 5 ==\n        print(f'Working on:\\n\\t>Block 5')\n\n        X = Conv2D(1024, (3, 3), strides=(2, 2), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X_sc = X\n\n        for layer_idx in tqdm(range(4)):\n            X = Conv2D(512, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Conv2D(1024, (3, 3), strides=(1, 1), padding='same')(X)\n            X = BatchNormalization()(X)\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X = Add()([X_sc, X])\n            X = LeakyReLU(alpha=0.1)(X)\n            \n            X_sc = X\n\n        # == OUTPUT ==\n        print(f'Working on:\\n\\t>Output layers')\n\n        X = Conv2D(512, (3, 3), strides=(1, 1), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X = Conv2D(256, (3, 3), strides=(1, 1), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        X = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(X)\n        X = BatchNormalization()(X)\n        X = LeakyReLU(alpha=0.1)(X)\n\n        preds = Conv2D(10, (1, 1), strides=(1, 1), activation='sigmoid')(X)\n\n        self.darknet_53 = Model(inputs=X_input, outputs=preds)\n\n        print(f'\\n===\\nModel was build successfully!\\n===\\n')\n\n    def compile_model(self, optimizer, loss):\n        self.darknet_53.compile(\n            optimizer=optimizer,\n            loss=loss\n        )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.2 - Auxiliary functions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"> One issue with yolo is that it is likely to contain more cells in its label grid that contain no objects than cells that do contain objects. It is easy then for the model to focus too much on learning to reduce no object cells to zero and not focus enough on getting the bounding boxes to the right shape. To overcome this the yolo paper suggests weighting the cells containing bounding boxes five times higher and the cells with no bounding boxes by half.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"- The loss Function performs three actions:\n> 1. Takes care of the confidance score that is trying to work out if the label grid cell contains a head of wheat or not\n> 2. Looks at the x, y positionof the bbox\n> 3. Looks at the width and height of the bbox","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_func(y_true, y_pred):\n    def _mask_by_y_true(y_true):\n        anchor_one_mask = tf.where(\n            y_true[..., 0]==0,\n            0.5,\n            5.0\n        )\n\n        anchor_two_mask = tf.where(\n            y_true[..., 5]==0,\n            0.5,\n            5.0\n        )\n    \n        return tf.concat([anchor_one_mask, anchor_two_mask], axis=0)\n\n    binary_crossentropy = prob_loss = BinaryCrossentropy(\n        reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE\n    )\n    \n    prob_loss = binary_crossentropy(\n        tf.concat([y_true[..., 0], y_true[..., 5]], axis=0),\n        tf.concat([y_pred[..., 0], y_pred[..., 5]], axis=0)\n    )\n\n    xy_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[..., 1:3], y_true[..., 6:8]], axis=0),\n        tf.concat([y_pred[..., 1:3], y_pred[..., 6:8]], axis=0)\n    )\n    \n    wh_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[..., 3:5], y_true[..., 8:10]], axis=0),\n        tf.concat([y_pred[..., 3:5], y_pred[..., 8:10]], axis=0)\n    )\n    \n    bboxes_mask = _mask_by_y_true(y_true)\n    \n    xy_loss = xy_loss * bboxes_mask\n    wh_loss = wh_loss * bboxes_mask\n    \n    return prob_loss + xy_loss + wh_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.3 - Main","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"yolo_v3 = YOLOv3()\nyolo_v3.darknet_53.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yolo_v3.compile_model(\n    optimizer=Adam(learning_rate=1e-4),\n    loss=loss_func\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5 - Train","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 5.1 - Callbacks","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"- Three callbacks are used:\n> - A high learning rate might be helpfull in the begining, but should be reduced as the model fits the data, so a call back that will reduce the learning rate when the loss does not decrease for two consecutive runs is added.\n> - An early stopping of the training and the restoration of the best weights in case the loss does not inprove in 5 consecutive iterations\n> - Tensor board for logging the train / val procedure","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=2, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1, restore_best_weights=True),\n    tf.keras.callbacks.TensorBoard(log_dir='./logs', write_graph=True, write_images=True, update_freq='epoch')\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5.2 - Model fitting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 80\nhistory = yolo_v3.darknet_53.fit(\n    train_generator,\n    validation_data=val_generator,\n    epochs=epochs,\n    callbacks=callbacks\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6 - Submission","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"To produce the submission file, we need to transform models' output from label grid of shape (16, 16, 4) to (m, 4) where m is the number of bounding boxes with high confidance.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 6.1 - Auxiliary functions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"- This function transforms each bounding box from a YOLO representation back to the COCO format, by:\n> 1. Reshape the bounding box from 0-1 scale back to 0-size of the image (256 in our case).\n> 2. Change the (x, y) coordinate from the center of the bounding box to the top left corner.\n> 3. Change the width and height back to x_max, y_max (i.e., VOC shape).\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_yolo_2_coco(yolo_bboxes, image_grid):\n    bboxes = yolo_bboxes.copy()\n    \n    im_width = (image_grid[:,:,2] * 32)\n    im_height = (image_grid[:,:,3] * 32)\n    \n    # descale x,y\n    bboxes[:,:,1] = np.floor(bboxes[:,:,1] * image_grid[:,:,2]) + image_grid[:,:,0]\n    bboxes[:,:,2] = np.floor(bboxes[:,:,2] * image_grid[:,:,3]) + image_grid[:,:,1]\n    bboxes[:,:,6] = np.floor(bboxes[:,:,6] * image_grid[:,:,2]) + image_grid[:,:,0]\n    bboxes[:,:,7] = np.floor(bboxes[:,:,7] * image_grid[:,:,3]) + image_grid[:,:,1]\n    \n    # descale width,height\n    bboxes[:,:,3] = bboxes[:,:,3] * im_width \n    bboxes[:,:,4] = bboxes[:,:,4] * im_height\n    bboxes[:,:,8] = bboxes[:,:,8] * im_width \n    bboxes[:,:,9] = bboxes[:,:,9] * im_height\n    \n    # centre x,y to top left x,y\n    bboxes[:,:,1] = bboxes[:,:,1] - np.floor(bboxes[:,:,3] / 2)\n    bboxes[:,:,2] = bboxes[:,:,2] - np.floor(bboxes[:,:,4] / 2)\n    bboxes[:,:,6] = bboxes[:,:,6] - np.floor(bboxes[:,:,8] / 2)\n    bboxes[:,:,7] = bboxes[:,:,7] - np.floor(bboxes[:,:,9] / 2)\n    \n    # width,heigth to x_max,y_max\n    bboxes[:,:,3] = bboxes[:,:,1] + bboxes[:,:,3]\n    bboxes[:,:,4] = bboxes[:,:,2] + bboxes[:,:,4]\n    bboxes[:,:,8] = bboxes[:,:,6] + bboxes[:,:,8]\n    bboxes[:,:,9] = bboxes[:,:,7] + bboxes[:,:,9]\n    \n    return bboxes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- The bounding boxes with low confidance should be removed","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def clear_low_conf_bboxes(preds, top_n):\n    def _switch_x_y(bboxes):\n        x1 = bboxes[:, 0].copy()\n        y1 = bboxes[:, 1].copy()\n        x2 = bboxes[:, 2].copy()\n        y2 = bboxes[:, 3].copy()\n        \n        bboxes[:, 0] = y1\n        bboxes[:, 1] = x1\n        bboxes[:, 2] = y2\n        bboxes[:, 3] = x2\n        \n        return bboxes\n    \n    def _top_n_preds(probs, bboxes, top_n):\n        bboxes = _switch_x_y(bboxes)\n        top_n_indices = tf.image.non_max_suppression(\n            boxes=bboxes,\n            scores=probs,\n            max_output_size=top_n,\n            iou_threshold=0.3,\n            score_threshold=0.3\n        ).numpy()\n        bboxes = _switch_x_y(bboxes)\n        \n        bboxes[:, 2:4] = bboxes[:, 2:4] - bboxes[:, 0:2]\n        \n        top_n_preds = list(zip(probs[top_n_indices], bboxes[top_n_indices]))\n    \n        res =  np.array(list(map(lambda pred: np.concatenate([[pred[0]], pred[1]]), top_n_preds)))\n\n        return res\n    \n    \n    probs = np.concatenate((preds[:, :, 0].flatten(), preds[:, :, 5].flatten()), axis=None)\n    \n    first_anchors = preds[:, :, 1:5].reshape((32*32, 4))\n    second_anchors = preds[:, :, 6:10].reshape((32*32, 4))\n    \n    bboxes = np.concatenate((first_anchors, second_anchors), axis=0)\n    \n    preds = _top_n_preds(probs, bboxes, top_n=top_n)\n\n    return preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Finla post processing function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_final_preds(yolo_bboxes, image_ids, image_grid):\n    preds = {}\n    coco_bboxes = yolo_bboxes.copy()\n    \n    for idx, img_id in enumerate(image_ids):\n        \n        coco_bboxes[idx] = format_yolo_2_coco(\n            yolo_bboxes=yolo_bboxes[idx], \n            image_grid=image_grid\n        )\n\n        preds[img_id] = clear_low_conf_bboxes(\n            preds=coco_bboxes[idx], \n            top_n=100\n        )\n    return preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6.2 - Main","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"- Load the test images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = data_root_dir / 'test'\ntest_dir.is_dir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_ids = os.listdir(test_dir)\ntest_image_ids = [img_id[:-4] for img_id in test_image_ids]\nprint(f'Test image ids:')\nfor img_id in test_image_ids:\n    print(f'\\t- {img_id}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = []\n\nfor idx, img_id in enumerate(test_image_ids):\n    img = Image.open(str(test_dir) + f'/{img_id}' + '.jpg')\n    img = img.resize((256, 256))\n    \n    img = np.asarray(img)\n    \n    aug = A.Compose([A.CLAHE(p=1), A.ToGray(p=1)])\n    \n    aug_img = aug(image=img)\n    \n    img = np.array(aug_img['image']) / 255\n    \n    img = np.expand_dims(img, axis=0)\n    \n    pred_yolo_bboxes = yolo_v3.darknet_53.predict(img)\n    \n    test_preds.append(pred_yolo_bboxes)\n    \ntest_yolo_preds = np.concatenate(test_preds)\ntest_coco_preds = get_final_preds(\n    yolo_bboxes=test_yolo_preds, \n    image_ids=test_image_ids, \n    image_grid=image_grid\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = {}\nmodel_scale = 256\noriginal_scale = 1024\nfor key in test_coco_preds:\n    pred_line = ''\n    for bbox_idx, bbox_pred in enumerate(test_coco_preds[key]):\n        for idx, pred in enumerate(bbox_pred):\n            if not idx:\n                pred_line += str(pred)\n            else:\n                pred_line += str(int(pred * original_scale / model_scale))\n                \n            if idx < len(bbox_pred):\n                pred_line += ' '\n\n#     print(f'line = {pred_line}')\n    test_preds[key] = pred_line\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds_df = pd.DataFrame(dict(image_id=list(test_preds.keys()), PredictionString=list(test_preds.values())))\nfinal_preds_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Save submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Save model weights","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"yolo_v3.darknet_53.save_weights(f'yolov3_{epochs}_epochs_weights')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6.3 - Show Predictions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### 6.3.1 - Auxiliary Functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_bboxes(bboxes_csv_file, model_scale, original_scale):\n    def _parse_str_line(str_line):\n        bboxes = []\n        data_line = np.array(str_line.split(' '))\n        print(data_line)\n        for idx, data in enumerate(data_line):\n            bbox_num_data = []\n            start_idx = idx * 5\n            if start_idx >= len(data_line) - 5:\n                break\n            # print(f'{start_idx}, {len(data_line) + 5}')\n            data_idx = np.arange(start_idx, start_idx + 5)\n            # print(data_idx)\n            bbox_str_data = data_line[data_idx]\n            bbox_num_data.append(bbox_str_data[0])\n            for bbox_str in bbox_str_data[1:]:\n                bbox_num_data.append(int(bbox_str) * original_scale / model_scale)\n            # print(bbox_data)\n            bboxes.append(bbox_num_data)\n        return np.array(bboxes, dtype=np.float32)\n    bbox_df = pd.read_csv(bboxes_csv_file)\n    bbox_df.PredictionString = bbox_df.PredictionString.apply(_parse_str_line)\n    bbox_df.rename(columns={'PredictionString': 'PredictionArray'}, inplace=True)\n    return bbox_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_images(data_dir, image_ids):\n    images = {}\n    for image_id in image_ids:\n        images[image_id] = np.asarray(Image.open(str(data_dir / image_id) + '.jpg'))\n    return images\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_test_image_sample(images, bboxes):\n    def _image_bbox_viz(ax, image, image_bboxes):\n        ax.imshow(image)\n        \n        for image_bbox in image_bboxes:\n            c, x, y, w, h = image_bbox\n            print(f'c = {c}, x = {x}, y = {y}, w = {w}, h = {h}')\n            ax.add_patch(Rectangle((x, y), w, h, fill=False, lw=1.5, color='red'))\n            \n        return np.asarray(ax)\n    \n    fig, axs = plt.subplots(1, len(images), figsize=(200, 200))\n\n    for idx, image_bbox in enumerate(zip(images, bboxes)):\n        _image_bbox_viz(axs[idx], image_bbox[0], image_bbox[1])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 6.3.2 - Main","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"bbox_preds_data_df = extract_bboxes(\n    bboxes_csv_file='submission.csv', \n    model_scale=1, \n    original_scale=1\n)\nbbox_preds_data_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimages = load_images(data_dir=Path(data_root_dir / 'test'), image_ids=bbox_preds_data_df.image_id.values)\nshow_test_image_sample(\n    images=images.values(), \n    bboxes=bbox_preds_data_df.PredictionArray.values\n)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}