{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import Input, Model\nfrom tensorflow.keras.layers import Conv2D, BatchNormalization, LeakyReLU, ReLU, MaxPool2D, Add, Dense, Dropout, Flatten, GlobalAveragePooling2D\nimport tensorflow.compat.v1 as tf1\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\n\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\n\nimport numpy as np\n\nimport PIL\nfrom PIL import Image, ImageDraw, ImageEnhance\n\nimport albumentations as albu\n\nfrom tqdm.auto import tqdm\n\nimport random\nrandom.seed(42)\n\nfrom warnings import filterwarnings\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# config = tf1.ConfigProto()\n# config.gpu_options.allow_growth=True\n# sess = tf1.Session(config=config)\n# tf1.keras.backend.set_session(session=sess)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# gpu_options = tf1.GPUOptions(per_process_gpu_memory_fraction=0.9999)\n# sess = tf1.Session(config=tf1.ConfigProto(gpu_options=gpu_options))\n# tf1.keras.backend.set_session(session=sess)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"policy = mixed_precision.Policy('mixed_bfloat16')\nmixed_precision.set_policy(policy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# OpenCV with GPU variable\nuse_GPU = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/global-wheat-detection/train.csv\")\ndf_train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id_values = df_train[\"image_id\"].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(image_id_values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_ids = image_id_values[0:3363]\nval_image_ids = image_id_values[3363:3373]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def group_boxes(group):\n    boundaries = group['bbox'].str.split(',', expand=True)\n    boundaries[0] = boundaries[0].str.slice(start=1)\n    boundaries[3] = boundaries[3].str.slice(stop=-1)\n    \n    return boundaries.values.astype(float)\n\nbboxes = df_train.groupby('image_id').apply(group_boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bboxes['bce2fdc4d'][:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(image_id):\n    \n    global use_GPU\n    if use_GPU:\n        image = cv2.UMat(cv2.imread('../input/global-wheat-detection/train/'+image_id+'.jpg'))\n    else:\n        image = cv2.imread('../input/global-wheat-detection/train/'+image_id+'.jpg')\n        \n    if use_GPU:\n        if len(cv2.UMat.get(image)) == 0:\n            raise ValueError(f\"Image could not be located\")\n    else:\n        if len(image) == 0:\n            raise ValueError(f\"Image could not be located\")\n    image = cv2.resize(image, (256, 256))\n    \n    if use_GPU:\n        return cv2.UMat.get(image)\n    else:\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(image_id):\n    image = Image.open('../input/global-wheat-detection/train/' + image_id + \".jpg\")\n    image = image.resize((256, 256))\n    \n    return np.asarray(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pixels = {}\ntrain_labels = {}\n\nfor image_id in tqdm(train_image_ids):\n    \n    train_pixels[image_id] = load_image(image_id)\n    train_labels[image_id] = bboxes[image_id].copy() / 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_pixels = {}\nval_labels = {}\n\nfor image_id in tqdm(val_image_ids):\n    val_pixels[image_id] = load_image(image_id)\n    val_labels[image_id] = bboxes[image_id].copy() / 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(train_pixels['b6ab77fd7'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_labels['b6ab77fd7'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_bboxes(image_id, bboxes, source='train'):\n    if use_GPU:\n        image = cv2.UMat(cv2.imread('../input/global-wheat-detection/'+source+'/'+image_id+'.jpg'))\n    else:\n        image = cv2.imread('../input/global-wheat-detection/'+source+'/'+image_id+'.jpg')\n    image = cv2.resize(image, (256, 256))\n    \n#     cv2.imshow('image', image)\n#     cv2.waitKey()\n#     cv2.destroyAllWindows()\n    \n    for bbox in bboxes:\n        image = draw_bbox(image, bbox)\n        \n    return image\ndef draw_bbox(image, bbox):\n    x, y, w, h = bbox\n    image = cv2.rectangle(image,\n                          (int(x), int(y)),\n                          (int(x+w), int(y+h)),\n                          (225,0,0), \n                          1)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_images(image_ids, bboxes, source='train'):\n    pixels = []\n    global use_GPU\n    \n    for image_id in image_ids:\n        pixels.append(\n            draw_bboxes(image_id, bboxes[image_id], source)\n        )\n        \n    num_of_images = len(image_ids)\n    fig, axes = plt.subplots(1, num_of_images, figsize=(10*num_of_images, 10*num_of_images))\n    \n    for i, image_pixels in enumerate(pixels):\n        if use_GPU:\n            axes[i].imshow(cv2.UMat.get(image_pixels))\n        else:\n            axes[i].imshow(image_pixels)\n    #plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Images with Bounding Boxes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nshow_images((train_image_ids[100:102]),(train_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_grid_lines(image_id, labels, grid_line_distance=32, source='train'):\n    global use_GPU\n    \n    image_path = '../input/global-wheat-detection/'+source+'/'+image_id+'.jpg'\n    if use_GPU:\n        image = cv2.UMat(draw_bboxes(image_id, labels))\n    else:\n        image = draw_bboxes(image_id, labels)\n    image = cv2.resize(image, (256, 256))\n    \n    # Vertical Lines\n    prev_x = 0\n    prev_y = 0\n    for dist in range(0, 256, grid_line_distance):\n        image = cv2.line(image, (prev_x, prev_y+(dist-prev_y)), (prev_x+256, prev_y+(dist-prev_y)), (0, 0, 255), 1, 1)\n        prev_x = prev_x\n        prev_y = dist\n    \n    # Horizontal Lines\n    prev_x = 0\n    prev_y = 0\n    for dist in range(0, 256, grid_line_distance):\n        image = cv2.line(image, (prev_x+(dist-prev_x), prev_y), (prev_x+(dist-prev_x), prev_y+256), (0, 0, 255), 1, 1)\n        prev_x = dist\n        prev_y = prev_y\n        \n    if use_GPU:\n        image = cv2.UMat.get(image)\n        \n    else:\n        image = image\n\n    return image\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Which grid size is optimal to select for Yolo-Similar model with an input image size of `256*256`?\nAnswer: below the plot","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10*2, 10*2))\nimage_id = train_image_ids[3000]\nax[0].set_title('Grid size: 32*32')\nax[0].imshow(draw_grid_lines(image_id, train_labels[image_id], grid_line_distance=32 )) # 32*32 grid\nax[1].set_title('Grid size: 16*16')\nax[1].imshow(draw_grid_lines(image_id, train_labels[image_id], grid_line_distance=16 )) # 16*16 grid\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Here we can see that `32*32` grid is more effective than `16*16` \n2. `16*16` grid is too closely packed and the mid-points of the anchors are not clear, Therefore, selecting `32*32` grid size is optimal.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_ids[100]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cleaning Very small boxes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tiny_boxes = []\n\nfor i, image_id in enumerate(train_image_ids):\n    for label in train_labels[image_id]:\n        if label[2]*label[3] <= 10 and label[2]*label[3] != 0:\n            tiny_boxes.append(i)\nprint(str(len(tiny_boxes)) + ' tiny boxes found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_labels(train_image_ids, train_labels):\n    good_labels = {}\n    \n    for i, image_id in enumerate(train_image_ids):\n        good_labels[image_id] = []\n        \n        for j, label in enumerate(train_labels[image_id]):\n            \n            if label[2]*label[3] > 8000 and i not in [1079, 1371, 2020]:\n                continue\n                \n            elif label[2]<5 or label[3]<5:\n                continue\n                \n            else:\n                good_labels[image_id].append(\n                    train_labels[image_id][j]\n                )\n    return good_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = clean_labels(train_image_ids, train_labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataGenerator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n\n    def __init__(self, image_ids, image_pixels, labels=None, batch_size=1, shuffle=False, augment=False):\n        self.image_ids = image_ids\n        self.image_pixels = image_pixels\n        self.labels = labels\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.on_epoch_end()\n        \n        self.image_grid = self.form_image_grid()\n        \n        \n    def form_image_grid(self):    \n        image_grid = np.zeros((32, 32, 4))\n\n        # x, y, width, height\n        cell = [0, 0, 256 / 32, 256 / 32] \n\n        for i in range(0, 32):\n            for j in range(0, 32):\n                image_grid[i,j] = cell\n\n                cell[0] = cell[0] + cell[2]\n\n            cell[0] = 0\n            cell[1] = cell[1] + cell[3]\n\n        return image_grid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def __len__(self):\n    return int(np.floor(len(self.image_ids) / self.batch_size))\n\n\ndef on_epoch_end(self):\n    self.indexes = np.arange(len(self.image_ids))\n\n    if self.shuffle == True:\n        np.random.shuffle(self.indexes)\n\n\nDataGenerator.__len__ = __len__\nDataGenerator.on_epoch_end = on_epoch_end","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DataGenerator.train_augmentations = albu.Compose([\n        albu.RandomSizedCrop(\n            min_max_height=(200, 200), \n            height=256, \n            width=256, \n            p=0.8\n        ),\n        albu.OneOf([\n            albu.Flip(),\n            albu.RandomRotate90(),\n        ], p=1),\n        albu.OneOf([\n            albu.HueSaturationValue(),\n            albu.RandomBrightnessContrast()\n        ], p=1),\n        albu.OneOf([\n            albu.GaussNoise(),\n            albu.GlassBlur(),\n            albu.ISONoise(),\n            albu.MultiplicativeNoise(),\n        ], p=0.5),\n        albu.Cutout(\n            num_holes=8, \n            max_h_size=16, \n            max_w_size=16, \n            fill_value=0, \n            p=0.5\n        ),\n        albu.CLAHE(p=1),\n        albu.ToGray(p=1),\n    ], \n    bbox_params={'format': 'coco', 'label_fields': ['labels']})\n\nDataGenerator.val_augmentations = albu.Compose([\n    albu.CLAHE(p=1),\n    albu.ToGray(p=1),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def __getitem__(self, index):\n    indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]\n\n    batch_ids = [self.image_ids[i] for i in indexes]\n\n    X, y = self.__data_generation(batch_ids)\n\n    return X, y\n\n\ndef __data_generation(self, batch_ids):\n    X, y = [], []\n\n    # Generate data\n    for i, image_id in enumerate(batch_ids):\n        pixels = self.image_pixels[image_id]\n        bboxes = self.labels[image_id]\n\n        if self.augment:     \n            pixels, bboxes = self.augment_image(pixels, bboxes)\n        else:\n            pixels = self.contrast_image(pixels)\n            bboxes = self.form_label_grid(bboxes)\n\n        X.append(pixels)\n        y.append(bboxes)\n\n    return np.array(X), np.array(y)\n\n\ndef augment_image(self, pixels, bboxes):\n    bbox_labels = np.ones(len(bboxes))\n\n    aug_result = self.train_augmentations(image=pixels, bboxes=bboxes, labels=bbox_labels)\n\n    bboxes = self.form_label_grid(aug_result['bboxes'])\n\n    return np.array(aug_result['image']) / 255, bboxes\n\n\ndef contrast_image(self, pixels):        \n    aug_result = self.val_augmentations(image=pixels)\n    return np.array(aug_result['image']) / 255\n\n\nDataGenerator.__getitem__ = __getitem__\nDataGenerator.__data_generation = __data_generation\nDataGenerator.augment_image = augment_image\nDataGenerator.contrast_image = contrast_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def form_label_grid(self, bboxes):\n    label_grid = np.zeros((32, 32, 10))\n\n    for i in range(0, 32):\n        for j in range(0, 32):\n            cell = self.image_grid[i,j]\n            label_grid[i,j] = self.rect_intersect(cell, bboxes)\n\n    return label_grid\n\n\ndef rect_intersect(self, cell, bboxes): \n    cell_x, cell_y, cell_width, cell_height = cell\n    cell_x_max = cell_x + cell_width \n    cell_y_max = cell_y + cell_height\n    \n    anchor_one = np.array([0, 0, 0, 0, 0])\n    anchor_two = np.array([0, 0, 0, 0, 0])\n\n    # check all boxes\n    for bbox in bboxes:\n        box_x, box_y, box_width, box_height = bbox\n        box_x_centre = box_x + (box_width / 2)\n        box_y_centre = box_y + (box_height / 2)\n\n        if(box_x_centre >= cell_x and box_x_centre < cell_x_max and box_y_centre >= cell_y and box_y_centre < cell_y_max):\n            \n            if anchor_one[0] == 0:\n                anchor_one = self.yolo_shape(\n                    [box_x, box_y, box_width, box_height], \n                    [cell_x, cell_y, cell_width, cell_height]\n                )\n            \n            elif anchor_two[0] == 0:\n                anchor_two = self.yolo_shape(\n                    [box_x, box_y, box_width, box_height], \n                    [cell_x, cell_y, cell_width, cell_height]\n                )\n                \n            else:\n                break\n\n    return np.concatenate((anchor_one, anchor_two), axis=None)\n\n\ndef yolo_shape(self, box, cell):\n    box_x, box_y, box_width, box_height = box\n    cell_x, cell_y, cell_width, cell_height = cell\n\n    # top left x,y to centre x,y\n    box_x = box_x + (box_width / 2)\n    box_y = box_y + (box_height / 2)\n\n    # offset bbox x,y to cell x,y\n    box_x = (box_x - cell_x) / cell_width\n    box_y = (box_y - cell_y) / cell_height\n\n    # bbox width,height relative to cell width,height\n    box_width = box_width / 256\n    box_height = box_height / 256\n\n    return [1, box_x, box_y, box_width, box_height]\n\n\nDataGenerator.form_label_grid = form_label_grid\nDataGenerator.rect_intersect = rect_intersect\nDataGenerator.yolo_shape = yolo_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 6\n\ntrain_generator = DataGenerator(\n    train_image_ids,\n    train_pixels,\n    train_labels, \n    batch_size=BATCH_SIZE, \n    shuffle=True,\n    augment=True\n)\n\nval_generator = DataGenerator(\n    val_image_ids, \n    val_pixels,\n    val_labels, \n    batch_size=10,\n    shuffle=False,\n    augment=False\n)\n\nimage_grid = train_generator.image_grid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Using Kaggle TPU for taining.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### For Using Kaggle TPU for taining.\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\n\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_main_block(num_times_to_add_block: int, x, x_shortcut, filters: int, kernel_size: tuple, strides: tuple, block_number=None, padding: str ='same', alpha: str = 0.1):\n    \n    for _ in range(num_times_to_add_block):\n        x = Conv2D(filters, kernel_size, strides=strides, padding=padding)(x)\n        x = BatchNormalization()(x)\n        x = LeakyReLU(alpha=alpha)(x)\n    \n        x = Conv2D(filters*2, kernel_size, strides=strides, padding=padding)(x)\n        x = BatchNormalization()(x)\n        x = LeakyReLU(alpha=alpha)(x)\n    \n        x = Add()([x_shortcut, x])\n        x = LeakyReLU(alpha=alpha)(x)\n        \n        x_shortcut = x\n        \n    return x, x_shortcut\n\n\n\ndef add_res_block(x, x_shortcut=None, filters: int = None, kernel_size: tuple = None, strides: tuple = None, block_number=None, padding: str = 'same', alpha: float = 0.1):\n    x = tf.keras.layers.Conv2D(filters, kernel_size, strides=strides, padding=padding)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=alpha)(x)\n\n    x_shortcut = x\n    \n    return x, x_shortcut\n\n\ndef custom_loss(y_true, y_pred):\n    \n    global BATCH_SIZE\n    \n    #------------- For GPU Computing ----------------#\n#     binary_crossentropy = prob_loss = tf.keras.losses.BinaryCrossentropy(\n#         reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE\n#     )\n\n#     prob_loss = binary_crossentropy(\n#         tf.concat([y_true[:,:,:,0], y_true[:,:,:,5]], axis=0), \n#         tf.concat([y_pred[:,:,:,0], y_pred[:,:,:,5]], axis=0)\n#     )\n    #-------------- End of GPU Computing ---------------#\n    \n    \n    \n    #------------ For TPU Computing ------------#\n    binary_crossentropy = prob_loss = tf.keras.losses.BinaryCrossentropy(\n        reduction=tf.keras.losses.Reduction.NONE)\n    \n    # Binary Cross Entropy loss for TPU Computing.\n    reduce_sum = tf.reduce_sum(binary_crossentropy(\n        tf.concat([y_true[:,:,:,0], y_true[:,:,:,5]], axis=0),\n        tf.concat([y_pred[:,:,:,0], y_pred[:,:,:,5]], axis=0)\n    ))\n    \n    prob_loss = reduce_sum * (1. / BATCH_SIZE)\n    #-------------- End of TPU Computing --------------#\n    \n    xy_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[:,:,:,1:3], y_true[:,:,:,6:8]], axis=0), \n        tf.concat([y_pred[:,:,:,1:3], y_pred[:,:,:,6:8]], axis=0)\n    )\n    \n    wh_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[:,:,:,3:5], y_true[:,:,:,8:10]], axis=0), \n        tf.concat([y_pred[:,:,:,3:5], y_pred[:,:,:,8:10]], axis=0)\n    )\n    \n    bboxes_mask = get_mask(y_true)\n    \n    xy_loss = xy_loss * bboxes_mask\n    wh_loss = wh_loss * bboxes_mask\n    \n    return prob_loss + xy_loss + wh_loss\n\n\ndef get_mask(y_true):\n    anchor_one_mask = tf.where(\n        y_true[:,:,:,0] == 0, \n        0.5, \n        5.0\n    )\n    \n    anchor_two_mask = tf.where(\n        y_true[:,:,:,5] == 0, \n        0.5, \n        5.0\n    )\n    \n    bboxes_mask = tf.concat(\n        [anchor_one_mask,anchor_two_mask],\n        axis=0\n    )\n    \n    return bboxes_mask\n\n\nwith tpu_strategy.scope():\n\n\n    x_input = Input(shape=(256, 256, 3))\n\n    x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(x_input)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n\n\n    #------------ Block 1 -------------#\n    #-- Res-block-1 --#\n    x, x_shortcut = add_res_block(x, filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n    #-- Main-Block-1 --#\n    x, x_shortcut = add_main_block(2, x, x_shortcut, filters=32, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n\n\n    #------------ Block 2 -------------#\n    #-- Res-block-2 --#\n    x, x_shortcut = add_res_block(x, x_shortcut=x_shortcut, filters=128, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n    #-- Main-Block 2 --#\n    x, x_shortcut = add_main_block(2, x, x_shortcut, filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n\n\n    #------------ Block 3 -------------#\n    #-- Res-block 3 --#\n    x, x_shortcut = add_res_block(x, x_shortcut, filters=256, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n    #-- Main-Block --#\n    x, x_shortcut = add_main_block(8, x, x_shortcut, filters=128, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n\n\n    #------------ Block 4 -------------#\n    #-- Res-Block 4 --#\n    x, x_shortcut = add_res_block(x, x_shortcut, filters=512, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n    #-- Main-Block 4 --#\n    x, x_shortcut = add_main_block(8, x, x_shortcut, filters=256, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n\n\n    #------------ Block 5 -------------#\n    #-- Res-Block 5 --#\n    x, x_shortcut = add_res_block(x, x_shortcut, filters=1024, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n    #-- Main-Block 5 --#\n    x, x_shortcut = add_main_block(4, x, x_shortcut, filters=512, kernel_size=(3, 3), strides=(1, 1), padding='same', alpha=0.1)\n\n\n    #------------ Output Layers -------------#\n    x = Conv2D(512, (3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n\n    x = Conv2D(256, (3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n\n    x = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n\n    predictions= x = Conv2D(10, (1, 1), strides=(1, 1), activation='sigmoid', dtype=\"float32\")(x)\n    \n    optimiser = tf.keras.optimizers.Adam(learning_rate=0.0001)\n    model = tf.keras.Model(inputs=x_input, outputs=predictions)\n    \n\n    model.compile(\n        optimizer=optimiser, \n        loss=custom_loss\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"model = Model(inputs=x_input, outputs=predictions)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"x_input = tf.keras.Input(shape=(256,256,3))\n\nx = tf.keras.layers.Conv2D(32, (3, 3), strides=(1, 1), padding='same')(x_input)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n########## block 1 ##########\nx = tf.keras.layers.Conv2D(64, (3, 3), strides=(1, 1), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx_shortcut = x\n\nfor i in range(2):\n    x = tf.keras.layers.Conv2D(32, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Conv2D(64, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Add()([x_shortcut, x])\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x_shortcut = x\n\n\n########## block 2 ##########\nx = tf.keras.layers.Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx_shortcut = x\n\nfor i in range(2):\n    x = tf.keras.layers.Conv2D(64, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Add()([x_shortcut, x])\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x_shortcut = x\n\n########## block 3 ##########\nx = tf.keras.layers.Conv2D(256, (3, 3), strides=(2, 2), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx_shortcut = x\n\nfor i in range(8):\n    x = tf.keras.layers.Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Add()([x_shortcut, x])\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x_shortcut = x\n\n    \n########## block 4 ##########\nx = tf.keras.layers.Conv2D(512, (3, 3), strides=(2, 2), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx_shortcut = x\n\nfor i in range(8):\n    x = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Add()([x_shortcut, x])\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x_shortcut = x\n\n########## block 5 ##########\nx = tf.keras.layers.Conv2D(1024, (3, 3), strides=(2, 2), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx_shortcut = x\n\nfor i in range(4):\n    x = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same')(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x = tf.keras.layers.Add()([x_shortcut, x])\n    x = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\n    x_shortcut = x\n\n########## output layers ##########\nx = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\nx = tf.keras.layers.Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.LeakyReLU(alpha=0.1)(x)\n\npredictions = tf.keras.layers.Conv2D(10, (1, 1), strides=(1, 1), activation='sigmoid')(x)\n\nmodel = tf.keras.Model(inputs=x_input, outputs=predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def custom_loss(y_true, y_pred):\n    binary_crossentropy = prob_loss = tf.keras.losses.BinaryCrossentropy(\n        reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE\n    )\n    \n    prob_loss = binary_crossentropy(\n        tf.concat([y_true[:,:,:,0], y_true[:,:,:,5]], axis=0), \n        tf.concat([y_pred[:,:,:,0], y_pred[:,:,:,5]], axis=0)\n    )\n    \n    xy_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[:,:,:,1:3], y_true[:,:,:,6:8]], axis=0), \n        tf.concat([y_pred[:,:,:,1:3], y_pred[:,:,:,6:8]], axis=0)\n    )\n    \n    wh_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[:,:,:,3:5], y_true[:,:,:,8:10]], axis=0), \n        tf.concat([y_pred[:,:,:,3:5], y_pred[:,:,:,8:10]], axis=0)\n    )\n    \n    bboxes_mask = get_mask(y_true)\n    \n    xy_loss = xy_loss * bboxes_mask\n    wh_loss = wh_loss * bboxes_mask\n    \n    return prob_loss + xy_loss + wh_loss\n\n\ndef get_mask(y_true):\n    anchor_one_mask = tf.where(\n        y_true[:,:,:,0] == 0, \n        0.5, \n        5.0\n    )\n    \n    anchor_two_mask = tf.where(\n        y_true[:,:,:,5] == 0, \n        0.5, \n        5.0\n    )\n    \n    bboxes_mask = tf.concat(\n        [anchor_one_mask,anchor_two_mask],\n        axis=0\n    )\n    \n    return bboxes_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimiser = tf.keras.optimizers.Adam(learning_rate=0.0001)\n\nmodel.compile(\n    optimizer=optimiser, \n    loss=custom_loss\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=2, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1, restore_best_weights=True),\n    keras.callbacks.ModelCheckpoint(filepath=\"Model_save_diff_user.h5\", save_best_only=True, monitor='loss', save_weights_only=True, verbose=1)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"model.load_weights(\"./Model_save_diff_user.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_json = model.to_json()\nwith open(\"Model_save_structure_diff_user.h5\", \"w\") as json_file:\n    json_file.write(model_json)\nmodel.save_weights(\"Model_save_diff_user.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(\n    train_generator,\n    validation_data=val_generator,\n    epochs=80,\n    callbacks=callbacks\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nimport itertools\n\ndef gen(): \n  for i in itertools.count(1): \n    yield (i, [1] * i) \n\ndataset = tf.data.Dataset.from_generator( \n     gen, \n     (tf.int64, tf.int64), \n     (tf.TensorShape([]), tf.TensorShape([None]))) \n\nlist(dataset.take(3).as_numpy_iterator())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from numba import cuda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = cuda.get_current_device()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device.reset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del cuda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}