{"cells":[{"metadata":{},"cell_type":"markdown","source":"Code Credit: https://github.com/kentaroy47/frcnn-from-scratch-with-keras Thank you kentaroy for you nicely documented github repo.\n\nHere I am going to train Faster RCNN with 90% of images datasets.Where all the required data preprocessing I have done in Part 1: EDA and Data Processing Kernal(https://www.kaggle.com/kishor1210/eda-and-data-processing)\n\nprerequisite: We need to create annotation.txt which you can find in part 1 kernal.\n              We need to download pretrained weight \n \nPretrained Weight : https://github.com/fchollet/deep-learning-models/releases/tag/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5\n\nPart 1: https://www.kaggle.com/kishor1210/eda-and-data-processing","execution_count":null},{"metadata":{"id":"Vvd4_cFsoFtT","outputId":"e6ab05e5-b50b-42fd-af87-4d7720e8c33f","trusted":true},"cell_type":"code","source":"from __future__ import division\nfrom __future__ import print_function\nfrom __future__ import absolute_import\nimport random\nimport pprint\nimport sys\nimport time\nimport numpy as np\nfrom optparse import OptionParser\nimport pickle\nimport math\nimport cv2\nimport copy\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport os\n\nfrom sklearn.metrics import average_precision_score\n\nfrom keras import backend as K\nfrom keras.optimizers import Adam, SGD, RMSprop\nfrom keras.layers import Flatten, Dense, Input, Conv2D, MaxPooling2D, Dropout\nfrom keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, TimeDistributed\nfrom keras.engine.topology import get_source_inputs\nfrom keras.utils import layer_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.objectives import categorical_crossentropy\n\nfrom keras.models import Model\nfrom keras.utils import generic_utils\nfrom keras.engine import Layer, InputSpec\nfrom keras import initializers, regularizers","execution_count":null,"outputs":[]},{"metadata":{"id":"J1rqiEVVKH0v"},"cell_type":"markdown","source":"#config","execution_count":null},{"metadata":{"id":"L7AxhwTpJvM4","trusted":true},"cell_type":"code","source":"class Config:\n\n\tdef __init__(self):\n\n\t\t# Print the process or not\n\t\tself.verbose = True\n\n\t\t# Name of base network\n\t\tself.network = 'vgg'\n\n\t\t# Setting for data augmentation\n\t\tself.use_horizontal_flips = False\n\t\tself.use_vertical_flips = False\n\t\tself.rot_90 = False\n\n\t\t# Anchor box scales\n    # Note that if im_size is smaller, anchor_box_scales should be scaled\n    # Original anchor_box_scales in the paper is [128, 256, 512]\n\t\tself.anchor_box_scales = [64, 128, 256] \n\n\t\t# Anchor box ratios\n\t\tself.anchor_box_ratios = [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]\n\n\t\t# Size to resize the smallest side of the image\n\t\t# Original setting in paper is 600. Set to 300 in here to save training time\n\t\tself.im_size = 300\n\n\t\t# image channel-wise mean to subtract\n\t\tself.img_channel_mean = [103.939, 116.779, 123.68]\n\t\tself.img_scaling_factor = 1.0\n\n\t\t# number of ROIs at once\n\t\tself.num_rois = 4\n\n\t\t# stride at the RPN (this depends on the network configuration)\n\t\tself.rpn_stride = 16\n\n\t\tself.balanced_classes = False\n\n\t\t# scaling the stdev\n\t\tself.std_scaling = 4.0\n\t\tself.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]\n\n\t\t# overlaps for RPN\n\t\tself.rpn_min_overlap = 0.3\n\t\tself.rpn_max_overlap = 0.7\n\n\t\t# overlaps for classifier ROIs\n\t\tself.classifier_min_overlap = 0.1\n\t\tself.classifier_max_overlap = 0.5\n\n\t\t# placeholder for the class mapping, automatically generated by the parser\n\t\tself.class_mapping = None\n\n\t\tself.model_path = None","execution_count":null,"outputs":[]},{"metadata":{"id":"Bj0LGtYKKM2D"},"cell_type":"markdown","source":"Parser the data from annotation file\n","execution_count":null},{"metadata":{"id":"ZArRGC0-KMOL","trusted":true},"cell_type":"code","source":"def get_data(input_path):\n  \"\"\"Parse the data from annotation file\n  \n  Args:\n    input_path: annotation file path\n      \n  Returns:\n\t\tall_data: list(filepath, width, height, list(bboxes))\n\t\tclasses_count: dict{key:class_name, value:count_num} \n\t\t\te.g. {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745}\n\t\tclass_mapping: dict{key:class_name, value: idx}\n\t\t\te.g. {'Car': 0, 'Mobile phone': 1, 'Person': 2}\n\t\"\"\"\n  found_bg = False\n  all_imgs = {}\n\n  classes_count = {}\n\n  class_mapping = {}\n\n  visualise = True\n\n  i = 1\n\t\n  with open(input_path,'r') as f:\n    print('Parsing annotation files')\n    \n    for line in f:\n      \n      # Print process\n      sys.stdout.write('\\r'+'idx=' + str(i))\n      i += 1\n      line_split = line.strip().split(',')\n\t\t\t# Make sure the info saved in annotation file matching the format (path_filename, x1, y1, x2, y2, class_name)\n\t\t\t# Note:\n\t\t\t#\tOne path_filename might has several classes (class_name)\n\t\t\t#\tx1, y1, x2, y2 are the pixel value of the origial image, not the ratio value\n\t\t\t#\t(x1, y1) top left coordinates; (x2, y2) bottom right coordinates\n\t\t\t#   x1,y1-------------------\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t---------------------x2,y2\n      (filename,x1,y1,x2,y2,class_name) = line_split\n      \n      if class_name not in classes_count:\n        classes_count[class_name] = 1\n      else:\n        classes_count[class_name] += 1\n\n      if class_name not in class_mapping:\n        if class_name == 'bg' and found_bg == False:\n          print('Found class name with special name bg. Will be treated as a background region (this is usually for hard negative mining).')\n          found_bg = True\n        class_mapping[class_name] = len(class_mapping)\n\n      if filename not in all_imgs:\n        all_imgs[filename] = {}\n\t\t\t\t\n        img = cv2.imread(filename)\n        (rows,cols) = img.shape[:2]\n        all_imgs[filename]['filepath'] = filename\n        all_imgs[filename]['width'] = cols\n        all_imgs[filename]['height'] = rows\n        all_imgs[filename]['bboxes'] = []\n\t\t\t\t# if np.random.randint(0,6) > 0:\n\t\t\t\t# \tall_imgs[filename]['imageset'] = 'trainval'\n\t\t\t\t# else:\n\t\t\t\t# \tall_imgs[filename]['imageset'] = 'test'\n\n      all_imgs[filename]['bboxes'].append({'class': class_name, 'x1': int(x1), 'x2': int(x2), 'y1': int(y1), 'y2': int(y2)})\n\n\n    all_data = []\n    for key in all_imgs:\n      all_data.append(all_imgs[key])\n\t\t\n\t\t# make sure the bg class is last in the list\n    if found_bg:\n      if class_mapping['bg'] != len(class_mapping) - 1:\n        key_to_switch = [key for key in class_mapping.keys() if class_mapping[key] == len(class_mapping)-1][0]\n        val_to_switch = class_mapping['bg']\n        class_mapping['bg'] = len(class_mapping) - 1\n        class_mapping[key_to_switch] = val_to_switch\n\t\t\n    return all_data, classes_count, class_mapping","execution_count":null,"outputs":[]},{"metadata":{"id":"CHcD5Gb6Kc0T"},"cell_type":"markdown","source":"Define ROI Pooling Convolutional Layer","execution_count":null},{"metadata":{"id":"LJv_J7ZlKXPe","trusted":true},"cell_type":"code","source":"class RoiPoolingConv(Layer):\n    '''ROI pooling layer for 2D inputs.\n    See Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition,\n    K. He, X. Zhang, S. Ren, J. Sun\n    # Arguments\n        pool_size: int\n            Size of pooling region to use. pool_size = 7 will result in a 7x7 region.\n        num_rois: number of regions of interest to be used\n    # Input shape\n        list of two 4D tensors [X_img,X_roi] with shape:\n        X_img:\n        `(1, rows, cols, channels)`\n        X_roi:\n        `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n    # Output shape\n        3D tensor with shape:\n        `(1, num_rois, channels, pool_size, pool_size)`\n    '''\n    def __init__(self, pool_size, num_rois, **kwargs):\n\n        self.dim_ordering = K.common.image_dim_ordering()\n        self.pool_size = pool_size\n        self.num_rois = num_rois\n\n        super(RoiPoolingConv, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.nb_channels = input_shape[0][3]   \n\n    def compute_output_shape(self, input_shape):\n        return None, self.num_rois, self.pool_size, self.pool_size, self.nb_channels\n\n    def call(self, x, mask=None):\n\n        assert(len(x) == 2)\n\n        # x[0] is image with shape (rows, cols, channels)\n        img = x[0]\n\n        # x[1] is roi with shape (num_rois,4) with ordering (x,y,w,h)\n        rois = x[1]\n\n        input_shape = K.shape(img)\n\n        outputs = []\n\n        for roi_idx in range(self.num_rois):\n\n            x = rois[0, roi_idx, 0]\n            y = rois[0, roi_idx, 1]\n            w = rois[0, roi_idx, 2]\n            h = rois[0, roi_idx, 3]\n\n            x = K.cast(x, 'int32')\n            y = K.cast(y, 'int32')\n            w = K.cast(w, 'int32')\n            h = K.cast(h, 'int32')\n\n            # Resized roi of the image to pooling size (7x7)\n            rs = tf.image.resize(img[:, y:y+h, x:x+w, :], (self.pool_size, self.pool_size))\n            outputs.append(rs)\n                \n\n        final_output = K.concatenate(outputs, axis=0)\n\n        # Reshape to (1, num_rois, pool_size, pool_size, nb_channels)\n        # Might be (1, 4, 7, 7, 3)\n        final_output = K.reshape(final_output, (1, self.num_rois, self.pool_size, self.pool_size, self.nb_channels))\n\n        # permute_dimensions is similar to transpose\n        final_output = K.permute_dimensions(final_output, (0, 1, 2, 3, 4))\n\n        return final_output\n    \n    \n    def get_config(self):\n        config = {'pool_size': self.pool_size,\n                  'num_rois': self.num_rois}\n        base_config = super(RoiPoolingConv, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"id":"1FWVzRo-Kl_d"},"cell_type":"markdown","source":"Vgg-16 model","execution_count":null},{"metadata":{"id":"GiLzeeuKKlUy","trusted":true},"cell_type":"code","source":"def get_img_output_length(width, height):\n    def get_output_length(input_length):\n        return input_length//16\n\n    return get_output_length(width), get_output_length(height)    \n\ndef nn_base(input_tensor=None, trainable=False):\n\n\n    input_shape = (None, None, 3)\n    if not K.is_keras_tensor(input_tensor):\n      img_input = Input(tensor=input_tensor, shape=input_shape)\n    else:\n      img_input = input_tensor\n\n    bn_axis = 3\n\n    # Block 1\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)\n\n    # Block 2\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)\n\n    # Block 3\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)\n\n    # Block 4\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)\n\n    # Block 5\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)\n    # x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)\n\n    return x","execution_count":null,"outputs":[]},{"metadata":{"id":"ev2O8oemMqPQ"},"cell_type":"markdown","source":"RPN layer","execution_count":null},{"metadata":{"id":"LutHJ2Z2KvIP","trusted":true},"cell_type":"code","source":"def rpn_layer(base_layers, num_anchors):\n    \"\"\"Create a rpn layer\n        Step1: Pass through the feature map from base layer to a 3x3 512 channels convolutional layer\n                Keep the padding 'same' to preserve the feature map's size\n        Step2: Pass the step1 to two (1,1) convolutional layer to replace the fully connected layer\n                classification layer: num_anchors (9 in here) channels for 0, 1 sigmoid activation output\n                regression layer: num_anchors*4 (36 in here) channels for computing the regression of bboxes with linear activation\n    Args:\n        base_layers: vgg in here\n        num_anchors: 9 in here\n\n    Returns:\n        [x_class, x_regr, base_layers]\n        x_class: classification for whether it's an object\n        x_regr: bboxes regression\n        base_layers: vgg in here\n    \"\"\"\n    x = Conv2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)\n\n    x_class = Conv2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)\n    x_regr = Conv2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)\n\n    return [x_class, x_regr, base_layers]","execution_count":null,"outputs":[]},{"metadata":{"id":"1XFLW0W7Owpm"},"cell_type":"markdown","source":"Classifier layer","execution_count":null},{"metadata":{"id":"WLj6gfP6MydC","trusted":true},"cell_type":"code","source":"def classifier_layer(base_layers, input_rois, num_rois, nb_classes = 4):\n    \"\"\"Create a classifier layer\n    \n    Args:\n        base_layers: vgg\n        input_rois: `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n        num_rois: number of rois to be processed in one time (4 in here)\n\n    Returns:\n        list(out_class, out_regr)\n        out_class: classifier layer output\n        out_regr: regression layer output\n    \"\"\"\n\n    input_shape = (num_rois,7,7,512)\n\n    pooling_regions = 7\n\n    # out_roi_pool.shape = (1, num_rois, channels, pool_size, pool_size)\n    # num_rois (4) 7x7 roi pooling\n    out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois])\n\n    # Flatten the convlutional layer and connected to 2 FC and 2 dropout\n    out = TimeDistributed(Flatten(name='flatten'))(out_roi_pool)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc1'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc2'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n\n    # There are two output layer\n    # out_class: softmax acivation function for classify the class name of the object\n    # out_regr: linear activation function for bboxes coordinates regression\n    out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out)\n    # note: no regression target for bg class\n    out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out)\n\n    return [out_class, out_regr]","execution_count":null,"outputs":[]},{"metadata":{"id":"uB_p3Nu2O4oP"},"cell_type":"markdown","source":"Calculate IoU (Intersection of Union)","execution_count":null},{"metadata":{"id":"dn_dBNtCO38-","trusted":true},"cell_type":"code","source":"def union(au, bu, area_intersection):\n\tarea_a = (au[2] - au[0]) * (au[3] - au[1])\n\tarea_b = (bu[2] - bu[0]) * (bu[3] - bu[1])\n\tarea_union = area_a + area_b - area_intersection\n\treturn area_union\n\n\ndef intersection(ai, bi):\n\tx = max(ai[0], bi[0])\n\ty = max(ai[1], bi[1])\n\tw = min(ai[2], bi[2]) - x\n\th = min(ai[3], bi[3]) - y\n\tif w < 0 or h < 0:\n\t\treturn 0\n\treturn w*h\n\n\ndef iou(a, b):\n\t# a and b should be (x1,y1,x2,y2)\n\n\tif a[0] >= a[2] or a[1] >= a[3] or b[0] >= b[2] or b[1] >= b[3]:\n\t\treturn 0.0\n\n\tarea_i = intersection(a, b)\n\tarea_u = union(a, b, area_i)\n\n\treturn float(area_i) / float(area_u + 1e-6)","execution_count":null,"outputs":[]},{"metadata":{"id":"JHoYow4sPXPb"},"cell_type":"markdown","source":"\nCalculate the rpn for all anchors of all images\n","execution_count":null},{"metadata":{"id":"wi8b8lQRPC_E","trusted":true},"cell_type":"code","source":"def calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function):\n\t\"\"\"(Important part!) Calculate the rpn for all anchors \n\t\tIf feature map has shape 38x50=1900, there are 1900x9=17100 potential anchors\n\t\n\tArgs:\n\t\tC: config\n\t\timg_data: augmented image data\n\t\twidth: original image width (e.g. 600)\n\t\theight: original image height (e.g. 800)\n\t\tresized_width: resized image width according to C.im_size (e.g. 300)\n\t\tresized_height: resized image height according to C.im_size (e.g. 400)\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\n\tReturns:\n\t\ty_rpn_cls: list(num_bboxes, y_is_box_valid + y_rpn_overlap)\n\t\t\ty_is_box_valid: 0 or 1 (0 means the box is invalid, 1 means the box is valid)\n\t\t\ty_rpn_overlap: 0 or 1 (0 means the box is not an object, 1 means the box is an object)\n\t\ty_rpn_regr: list(num_bboxes, 4*y_rpn_overlap + y_rpn_regr)\n\t\t\ty_rpn_regr: x1,y1,x2,y2 bunding boxes coordinates\n\t\"\"\"\n\tdownscale = float(C.rpn_stride) \n\tanchor_sizes = C.anchor_box_scales   # 128, 256, 512\n\tanchor_ratios = C.anchor_box_ratios  # 1:1, 1:2*sqrt(2), 2*sqrt(2):1\n\tnum_anchors = len(anchor_sizes) * len(anchor_ratios) # 3x3=9\n\n\t# calculate the output map size based on the network architecture\n\t(output_width, output_height) = img_length_calc_function(resized_width, resized_height)\n\n\tn_anchratios = len(anchor_ratios)    # 3\n\t\n\t# initialise empty output objectives\n\ty_rpn_overlap = np.zeros((output_height, output_width, num_anchors))\n\ty_is_box_valid = np.zeros((output_height, output_width, num_anchors))\n\ty_rpn_regr = np.zeros((output_height, output_width, num_anchors * 4))\n\n\tnum_bboxes = len(img_data['bboxes'])\n\n\tnum_anchors_for_bbox = np.zeros(num_bboxes).astype(int)\n\tbest_anchor_for_bbox = -1*np.ones((num_bboxes, 4)).astype(int)\n\tbest_iou_for_bbox = np.zeros(num_bboxes).astype(np.float32)\n\tbest_x_for_bbox = np.zeros((num_bboxes, 4)).astype(int)\n\tbest_dx_for_bbox = np.zeros((num_bboxes, 4)).astype(np.float32)\n\n\t# get the GT box coordinates, and resize to account for image resizing\n\tgta = np.zeros((num_bboxes, 4))\n\tfor bbox_num, bbox in enumerate(img_data['bboxes']):\n\t\t# get the GT box coordinates, and resize to account for image resizing\n\t\tgta[bbox_num, 0] = bbox['x1'] * (resized_width / float(width))\n\t\tgta[bbox_num, 1] = bbox['x2'] * (resized_width / float(width))\n\t\tgta[bbox_num, 2] = bbox['y1'] * (resized_height / float(height))\n\t\tgta[bbox_num, 3] = bbox['y2'] * (resized_height / float(height))\n\t\n\t# rpn ground truth\n\n\tfor anchor_size_idx in range(len(anchor_sizes)):\n\t\tfor anchor_ratio_idx in range(n_anchratios):\n\t\t\tanchor_x = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][0]\n\t\t\tanchor_y = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][1]\t\n\t\t\t\n\t\t\tfor ix in range(output_width):\t\t\t\t\t\n\t\t\t\t# x-coordinates of the current anchor box\t\n\t\t\t\tx1_anc = downscale * (ix + 0.5) - anchor_x / 2\n\t\t\t\tx2_anc = downscale * (ix + 0.5) + anchor_x / 2\t\n\t\t\t\t\n\t\t\t\t# ignore boxes that go across image boundaries\t\t\t\t\t\n\t\t\t\tif x1_anc < 0 or x2_anc > resized_width:\n\t\t\t\t\tcontinue\n\t\t\t\t\t\n\t\t\t\tfor jy in range(output_height):\n\n\t\t\t\t\t# y-coordinates of the current anchor box\n\t\t\t\t\ty1_anc = downscale * (jy + 0.5) - anchor_y / 2\n\t\t\t\t\ty2_anc = downscale * (jy + 0.5) + anchor_y / 2\n\n\t\t\t\t\t# ignore boxes that go across image boundaries\n\t\t\t\t\tif y1_anc < 0 or y2_anc > resized_height:\n\t\t\t\t\t\tcontinue\n\n\t\t\t\t\t# bbox_type indicates whether an anchor should be a target\n\t\t\t\t\t# Initialize with 'negative'\n\t\t\t\t\tbbox_type = 'neg'\n\n\t\t\t\t\t# this is the best IOU for the (x,y) coord and the current anchor\n\t\t\t\t\t# note that this is different from the best IOU for a GT bbox\n\t\t\t\t\tbest_iou_for_loc = 0.0\n\n\t\t\t\t\tfor bbox_num in range(num_bboxes):\n\t\t\t\t\t\t\n\t\t\t\t\t\t# get IOU of the current GT box and the current anchor box\n\t\t\t\t\t\tcurr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1_anc, y1_anc, x2_anc, y2_anc])\n\t\t\t\t\t\t# calculate the regression targets if they will be needed\n\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num] or curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\tcx = (gta[bbox_num, 0] + gta[bbox_num, 1]) / 2.0\n\t\t\t\t\t\t\tcy = (gta[bbox_num, 2] + gta[bbox_num, 3]) / 2.0\n\t\t\t\t\t\t\tcxa = (x1_anc + x2_anc)/2.0\n\t\t\t\t\t\t\tcya = (y1_anc + y2_anc)/2.0\n\n\t\t\t\t\t\t\t# x,y are the center point of ground-truth bbox\n\t\t\t\t\t\t\t# xa,ya are the center point of anchor bbox (xa=downscale * (ix + 0.5); ya=downscale * (iy+0.5))\n\t\t\t\t\t\t\t# w,h are the width and height of ground-truth bbox\n\t\t\t\t\t\t\t# wa,ha are the width and height of anchor bboxe\n\t\t\t\t\t\t\t# tx = (x - xa) / wa\n\t\t\t\t\t\t\t# ty = (y - ya) / ha\n\t\t\t\t\t\t\t# tw = log(w / wa)\n\t\t\t\t\t\t\t# th = log(h / ha)\n\t\t\t\t\t\t\ttx = (cx - cxa) / (x2_anc - x1_anc)\n\t\t\t\t\t\t\tty = (cy - cya) / (y2_anc - y1_anc)\n\t\t\t\t\t\t\ttw = np.log((gta[bbox_num, 1] - gta[bbox_num, 0]) / (x2_anc - x1_anc))\n\t\t\t\t\t\t\tth = np.log((gta[bbox_num, 3] - gta[bbox_num, 2]) / (y2_anc - y1_anc))\n\t\t\t\t\t\t\n\t\t\t\t\t\tif img_data['bboxes'][bbox_num]['class'] != 'bg':\n\n\t\t\t\t\t\t\t# all GT boxes should be mapped to an anchor box, so we keep track of which anchor box was best\n\t\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num]:\n\t\t\t\t\t\t\t\tbest_anchor_for_bbox[bbox_num] = [jy, ix, anchor_ratio_idx, anchor_size_idx]\n\t\t\t\t\t\t\t\tbest_iou_for_bbox[bbox_num] = curr_iou\n\t\t\t\t\t\t\t\tbest_x_for_bbox[bbox_num,:] = [x1_anc, x2_anc, y1_anc, y2_anc]\n\t\t\t\t\t\t\t\tbest_dx_for_bbox[bbox_num,:] = [tx, ty, tw, th]\n\n\t\t\t\t\t\t\t# we set the anchor to positive if the IOU is >0.7 (it does not matter if there was another better box, it just indicates overlap)\n\t\t\t\t\t\t\tif curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\t\tbbox_type = 'pos'\n\t\t\t\t\t\t\t\tnum_anchors_for_bbox[bbox_num] += 1\n\t\t\t\t\t\t\t\t# we update the regression layer target if this IOU is the best for the current (x,y) and anchor position\n\t\t\t\t\t\t\t\tif curr_iou > best_iou_for_loc:\n\t\t\t\t\t\t\t\t\tbest_iou_for_loc = curr_iou\n\t\t\t\t\t\t\t\t\tbest_regr = (tx, ty, tw, th)\n\n\t\t\t\t\t\t\t# if the IOU is >0.3 and <0.7, it is ambiguous and no included in the objective\n\t\t\t\t\t\t\tif C.rpn_min_overlap < curr_iou < C.rpn_max_overlap:\n\t\t\t\t\t\t\t\t# gray zone between neg and pos\n\t\t\t\t\t\t\t\tif bbox_type != 'pos':\n\t\t\t\t\t\t\t\t\tbbox_type = 'neutral'\n\n\t\t\t\t\t# turn on or off outputs depending on IOUs\n\t\t\t\t\tif bbox_type == 'neg':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'neutral':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'pos':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\tstart = 4 * (anchor_ratio_idx + n_anchratios * anchor_size_idx)\n\t\t\t\t\t\ty_rpn_regr[jy, ix, start:start+4] = best_regr\n\n\t# we ensure that every bbox has at least one positive RPN region\n\n\tfor idx in range(num_anchors_for_bbox.shape[0]):\n\t\tif num_anchors_for_bbox[idx] == 0:\n\t\t\t# no box with an IOU greater than zero ...\n\t\t\tif best_anchor_for_bbox[idx, 0] == -1:\n\t\t\t\tcontinue\n\t\t\ty_is_box_valid[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\ty_rpn_overlap[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\tstart = 4 * (best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3])\n\t\t\ty_rpn_regr[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], start:start+4] = best_dx_for_bbox[idx, :]\n\n\ty_rpn_overlap = np.transpose(y_rpn_overlap, (2, 0, 1))\n\ty_rpn_overlap = np.expand_dims(y_rpn_overlap, axis=0)\n\n\ty_is_box_valid = np.transpose(y_is_box_valid, (2, 0, 1))\n\ty_is_box_valid = np.expand_dims(y_is_box_valid, axis=0)\n\n\ty_rpn_regr = np.transpose(y_rpn_regr, (2, 0, 1))\n\ty_rpn_regr = np.expand_dims(y_rpn_regr, axis=0)\n\n\tpos_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1))\n\tneg_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 0, y_is_box_valid[0, :, :, :] == 1))\n\n\tnum_pos = len(pos_locs[0])\n\n\t# one issue is that the RPN has many more negative than positive regions, so we turn off some of the negative\n\t# regions. We also limit it to 256 regions.\n\tnum_regions = 256\n\n\tif len(pos_locs[0]) > num_regions/2:\n\t\tval_locs = random.sample(range(len(pos_locs[0])), len(pos_locs[0]) - num_regions/2)\n\t\ty_is_box_valid[0, pos_locs[0][val_locs], pos_locs[1][val_locs], pos_locs[2][val_locs]] = 0\n\t\tnum_pos = num_regions/2\n\n\tif len(neg_locs[0]) + num_pos > num_regions:\n\t\tval_locs = random.sample(range(len(neg_locs[0])), len(neg_locs[0]) - num_pos)\n\t\ty_is_box_valid[0, neg_locs[0][val_locs], neg_locs[1][val_locs], neg_locs[2][val_locs]] = 0\n\n\ty_rpn_cls = np.concatenate([y_is_box_valid, y_rpn_overlap], axis=1)\n\ty_rpn_regr = np.concatenate([np.repeat(y_rpn_overlap, 4, axis=1), y_rpn_regr], axis=1)\n\n\treturn np.copy(y_rpn_cls), np.copy(y_rpn_regr), num_pos","execution_count":null,"outputs":[]},{"metadata":{"id":"1qn0nd2RPllj"},"cell_type":"markdown","source":"Get new image size and augment the image\n","execution_count":null},{"metadata":{"id":"iUHFPELdPk5D","trusted":true},"cell_type":"code","source":"def get_new_img_size(width, height, img_min_side=300):\n\tif width <= height:\n\t\tf = float(img_min_side) / width\n\t\tresized_height = int(f * height)\n\t\tresized_width = img_min_side\n\telse:\n\t\tf = float(img_min_side) / height\n\t\tresized_width = int(f * width)\n\t\tresized_height = img_min_side\n\n\treturn resized_width, resized_height\n\ndef augment(img_data, config, augment=True):\n\tassert 'filepath' in img_data\n\tassert 'bboxes' in img_data\n\tassert 'width' in img_data\n\tassert 'height' in img_data\n\n\timg_data_aug = copy.deepcopy(img_data)\n\n\timg = cv2.imread(img_data_aug['filepath'])\n\n\tif augment:\n\t\trows, cols = img.shape[:2]\n\n\t\tif config.use_horizontal_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 1)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\tbbox['x1'] = cols - x2\n\n\t\tif config.use_vertical_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 0)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\tbbox['y1'] = rows - y2\n\n\t\tif config.rot_90:\n\t\t\tangle = np.random.choice([0,90,180,270],1)[0]\n\t\t\tif angle == 270:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 0)\n\t\t\telif angle == 180:\n\t\t\t\timg = cv2.flip(img, -1)\n\t\t\telif angle == 90:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 1)\n\t\t\telif angle == 0:\n\t\t\t\tpass\n\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tif angle == 270:\n\t\t\t\t\tbbox['x1'] = y1\n\t\t\t\t\tbbox['x2'] = y2\n\t\t\t\t\tbbox['y1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = cols - x1\n\t\t\t\telif angle == 180:\n\t\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\t\tbbox['x1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = rows - y2\n\t\t\t\telif angle == 90:\n\t\t\t\t\tbbox['x1'] = rows - y2\n\t\t\t\t\tbbox['x2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = x1\n\t\t\t\t\tbbox['y2'] = x2        \n\t\t\t\telif angle == 0:\n\t\t\t\t\tpass\n\n\timg_data_aug['width'] = img.shape[1]\n\timg_data_aug['height'] = img.shape[0]\n\treturn img_data_aug, img","execution_count":null,"outputs":[]},{"metadata":{"id":"e7bJ6bU8P5_8"},"cell_type":"markdown","source":"Generate the ground_truth anchors\n","execution_count":null},{"metadata":{"id":"LvE2hPCHPxWh","trusted":true},"cell_type":"code","source":"def get_anchor_gt(all_img_data, C, img_length_calc_function, mode='train'):\n\t\"\"\" Yield the ground-truth anchors as Y (labels)\n\t\t\n\tArgs:\n\t\tall_img_data: list(filepath, width, height, list(bboxes))\n\t\tC: config\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\t\tmode: 'train' or 'test'; 'train' mode need augmentation\n\n\tReturns:\n\t\tx_img: image data after resized and scaling (smallest size = 300px)\n\t\tY: [y_rpn_cls, y_rpn_regr]\n\t\timg_data_aug: augmented image data (original image with augmentation)\n\t\tdebug_img: show image for debug\n\t\tnum_pos: show number of positive anchors for debug\n\t\"\"\"\n\twhile True:\n\n\t\tfor img_data in all_img_data:\n\t\t\ttry:\n\n\t\t\t\t# read in image, and optionally add augmentation\n\n\t\t\t\tif mode == 'train':\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=True)\n\t\t\t\telse:\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=False)\n\n\t\t\t\t(width, height) = (img_data_aug['width'], img_data_aug['height'])\n\t\t\t\t(rows, cols, _) = x_img.shape\n\n\t\t\t\tassert cols == width\n\t\t\t\tassert rows == height\n\n\t\t\t\t# get image dimensions for resizing\n\t\t\t\t(resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n\t\t\t\t# resize the image so that smalles side is length = 300px\n\t\t\t\tx_img = cv2.resize(x_img, (resized_width, resized_height), interpolation=cv2.INTER_CUBIC)\n\t\t\t\tdebug_img = x_img.copy()\n\n\t\t\t\ttry:\n\t\t\t\t\ty_rpn_cls, y_rpn_regr, num_pos = calc_rpn(C, img_data_aug, width, height, resized_width, resized_height, img_length_calc_function)\n\t\t\t\texcept:\n\t\t\t\t\tcontinue\n\n\t\t\t\t# Zero-center by mean pixel, and preprocess image\n\n\t\t\t\tx_img = x_img[:,:, (2, 1, 0)]  # BGR -> RGB\n\t\t\t\tx_img = x_img.astype(np.float32)\n\t\t\t\tx_img[:, :, 0] -= C.img_channel_mean[0]\n\t\t\t\tx_img[:, :, 1] -= C.img_channel_mean[1]\n\t\t\t\tx_img[:, :, 2] -= C.img_channel_mean[2]\n\t\t\t\tx_img /= C.img_scaling_factor\n\n\t\t\t\tx_img = np.transpose(x_img, (2, 0, 1))\n\t\t\t\tx_img = np.expand_dims(x_img, axis=0)\n\n\t\t\t\ty_rpn_regr[:, y_rpn_regr.shape[1]//2:, :, :] *= C.std_scaling\n\n\t\t\t\tx_img = np.transpose(x_img, (0, 2, 3, 1))\n\t\t\t\ty_rpn_cls = np.transpose(y_rpn_cls, (0, 2, 3, 1))\n\t\t\t\ty_rpn_regr = np.transpose(y_rpn_regr, (0, 2, 3, 1))\n\n\t\t\t\tyield np.copy(x_img), [np.copy(y_rpn_cls), np.copy(y_rpn_regr)], img_data_aug, debug_img, num_pos\n\n\t\t\texcept Exception as e:\n\t\t\t\tprint(e)\n\t\t\t\tcontinue","execution_count":null,"outputs":[]},{"metadata":{"id":"8FlcNe-cQAID"},"cell_type":"markdown","source":"Define loss functions for all four outputs\n","execution_count":null},{"metadata":{"id":"GJKLxDNZP-4M","trusted":true},"cell_type":"code","source":"lambda_rpn_regr = 1.0\nlambda_rpn_class = 1.0\n\nlambda_cls_regr = 1.0\nlambda_cls_class = 1.0\n\nepsilon = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"id":"APQ-fIJ6QHBb","trusted":true},"cell_type":"code","source":"def rpn_loss_regr(num_anchors):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def rpn_loss_regr_fixed_num(y_true, y_pred):\n\n        # x is the difference between true value and predicted vaue\n        x = y_true[:, :, :, 4 * num_anchors:] - y_pred\n\n        # absolute value of x\n        x_abs = K.abs(x)\n\n        # If x_abs <= 1.0, x_bool = 1\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), tf.float32)\n\n        return lambda_rpn_regr * K.sum(\n            y_true[:, :, :, :4 * num_anchors] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :, :4 * num_anchors])\n\n    return rpn_loss_regr_fixed_num\n\n\ndef rpn_loss_cls(num_anchors):\n    \"\"\"Loss function for rpn classification\n    Args:\n        num_anchors: number of anchors (9 in here)\n        y_true[:, :, :, :9]: [0,1,0,0,0,0,0,1,0] means only the second and the eighth box is valid which contains pos or neg anchor => isValid\n        y_true[:, :, :, 9:]: [0,1,0,0,0,0,0,0,0] means the second box is pos and eighth box is negative\n    Returns:\n        lambda * sum((binary_crossentropy(isValid*y_pred,y_true))) / N\n    \"\"\"\n    def rpn_loss_cls_fixed_num(y_true, y_pred):\n\n            return lambda_rpn_class * K.sum(y_true[:, :, :, :num_anchors] * K.binary_crossentropy(y_pred[:, :, :, :], y_true[:, :, :, num_anchors:])) / K.sum(epsilon + y_true[:, :, :, :num_anchors])\n\n    return rpn_loss_cls_fixed_num\n\n\ndef class_loss_regr(num_classes):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def class_loss_regr_fixed_num(y_true, y_pred):\n        x = y_true[:, :, 4*num_classes:] - y_pred\n        x_abs = K.abs(x)\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), 'float32')\n        return lambda_cls_regr * K.sum(y_true[:, :, :4*num_classes] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :4*num_classes])\n    return class_loss_regr_fixed_num\n\n\ndef class_loss_cls(y_true, y_pred):\n    return lambda_cls_class * K.mean(categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))","execution_count":null,"outputs":[]},{"metadata":{"id":"ADQUF45AQLZx"},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"id":"DbI2W714QKZ5","trusted":true},"cell_type":"code","source":"def non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):\n    # code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/\n    # if there are no boxes, return an empty list\n\n    # Process explanation:\n    #   Step 1: Sort the probs list\n    #   Step 2: Find the larget prob 'Last' in the list and save it to the pick list\n    #   Step 3: Calculate the IoU with 'Last' box and other boxes in the list. If the IoU is larger than overlap_threshold, delete the box from list\n    #   Step 4: Repeat step 2 and step 3 until there is no item in the probs list \n    if len(boxes) == 0:\n        return []\n\n    # grab the coordinates of the bounding boxes\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    np.testing.assert_array_less(x1, x2)\n    np.testing.assert_array_less(y1, y2)\n\n    # if the bounding boxes integers, convert them to floats --\n    # this is important since we'll be doing a bunch of divisions\n    if boxes.dtype.kind == \"i\":\n        boxes = boxes.astype(\"float\")\n\n    # initialize the list of picked indexes\t\n    pick = []\n\n    # calculate the areas\n    area = (x2 - x1) * (y2 - y1)\n\n    # sort the bounding boxes \n    idxs = np.argsort(probs)\n\n    # keep looping while some indexes still remain in the indexes\n    # list\n    while len(idxs) > 0:\n        # grab the last index in the indexes list and add the\n        # index value to the list of picked indexes\n        last = len(idxs) - 1\n        i = idxs[last]\n        pick.append(i)\n\n        # find the intersection\n\n        xx1_int = np.maximum(x1[i], x1[idxs[:last]])\n        yy1_int = np.maximum(y1[i], y1[idxs[:last]])\n        xx2_int = np.minimum(x2[i], x2[idxs[:last]])\n        yy2_int = np.minimum(y2[i], y2[idxs[:last]])\n\n        ww_int = np.maximum(0, xx2_int - xx1_int)\n        hh_int = np.maximum(0, yy2_int - yy1_int)\n\n        area_int = ww_int * hh_int\n\n        # find the union\n        area_union = area[i] + area[idxs[:last]] - area_int\n\n        # compute the ratio of overlap\n        overlap = area_int/(area_union + 1e-6)\n\n        # delete all indexes from the index list that have\n        idxs = np.delete(idxs, np.concatenate(([last],\n            np.where(overlap > overlap_thresh)[0])))\n\n        if len(pick) >= max_boxes:\n            break\n\n    # return only the bounding boxes that were picked using the integer data type\n    boxes = boxes[pick].astype(\"int\")\n    probs = probs[pick]\n    return boxes, probs\n\ndef apply_regr_np(X, T):\n    \"\"\"Apply regression layer to all anchors in one feature map\n\n    Args:\n        X: shape=(4, 18, 25) the current anchor type for all points in the feature map\n        T: regression layer shape=(4, 18, 25)\n\n    Returns:\n        X: regressed position and size for current anchor\n    \"\"\"\n    try:\n        x = X[0, :, :]\n        y = X[1, :, :]\n        w = X[2, :, :]\n        h = X[3, :, :]\n\n        tx = T[0, :, :]\n        ty = T[1, :, :]\n        tw = T[2, :, :]\n        th = T[3, :, :]\n\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n\n        w1 = np.exp(tw.astype(np.float64)) * w\n        h1 = np.exp(th.astype(np.float64)) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n\n        x1 = np.round(x1)\n        y1 = np.round(y1)\n        w1 = np.round(w1)\n        h1 = np.round(h1)\n        return np.stack([x1, y1, w1, h1])\n    except Exception as e:\n        print(e)\n        return X\n    \ndef apply_regr(x, y, w, h, tx, ty, tw, th):\n    # Apply regression to x, y, w and h\n    try:\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n        w1 = math.exp(tw) * w\n        h1 = math.exp(th) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        w1 = int(round(w1))\n        h1 = int(round(h1))\n\n        return x1, y1, w1, h1\n\n    except ValueError:\n        return x, y, w, h\n    except OverflowError:\n        return x, y, w, h\n    except Exception as e:\n        print(e)\n        return x, y, w, h\n\ndef calc_iou(R, img_data, C, class_mapping):\n    \"\"\"Converts from (x1,y1,x2,y2) to (x,y,w,h) format\n\n    Args:\n        R: bboxes, probs\n    \"\"\"\n    bboxes = img_data['bboxes']\n    (width, height) = (img_data['width'], img_data['height'])\n    # get image dimensions for resizing\n    (resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n    gta = np.zeros((len(bboxes), 4))\n\n    for bbox_num, bbox in enumerate(bboxes):\n        # get the GT box coordinates, and resize to account for image resizing\n        # gta[bbox_num, 0] = (40 * (600 / 800)) / 16 = int(round(1.875)) = 2 (x in feature map)\n        gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))\n        gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))\n\n    x_roi = []\n    y_class_num = []\n    y_class_regr_coords = []\n    y_class_regr_label = []\n    IoUs = [] # for debugging only\n\n    # R.shape[0]: number of bboxes (=300 from non_max_suppression)\n    for ix in range(R.shape[0]):\n        (x1, y1, x2, y2) = R[ix, :]\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        x2 = int(round(x2))\n        y2 = int(round(y2))\n\n        best_iou = 0.0\n        best_bbox = -1\n        # Iterate through all the ground-truth bboxes to calculate the iou\n        for bbox_num in range(len(bboxes)):\n            curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])\n\n            # Find out the corresponding ground-truth bbox_num with larget iou\n            if curr_iou > best_iou:\n                best_iou = curr_iou\n                best_bbox = bbox_num\n\n        if best_iou < C.classifier_min_overlap:\n                continue\n        else:\n            w = x2 - x1\n            h = y2 - y1\n            x_roi.append([x1, y1, w, h])\n            IoUs.append(best_iou)\n\n            if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:\n                # hard negative example\n                cls_name = 'bg'\n            elif C.classifier_max_overlap <= best_iou:\n                cls_name = bboxes[best_bbox]['class']\n                cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0\n                cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0\n\n                cx = x1 + w / 2.0\n                cy = y1 + h / 2.0\n\n                tx = (cxg - cx) / float(w)\n                ty = (cyg - cy) / float(h)\n                tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))\n                th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))\n            else:\n                print('roi = {}'.format(best_iou))\n                raise RuntimeError\n\n        class_num = class_mapping[cls_name]\n        class_label = len(class_mapping) * [0]\n        class_label[class_num] = 1\n        y_class_num.append(copy.deepcopy(class_label))\n        coords = [0] * 4 * (len(class_mapping) - 1)\n        labels = [0] * 4 * (len(class_mapping) - 1)\n        if cls_name != 'bg':\n            label_pos = 4 * class_num\n            sx, sy, sw, sh = C.classifier_regr_std\n            coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]\n            labels[label_pos:4+label_pos] = [1, 1, 1, 1]\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n        else:\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n\n    if len(x_roi) == 0:\n        return None, None, None, None\n\n    # bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n    X = np.array(x_roi)\n    # one hot code for bboxes from above => x_roi (X)\n    Y1 = np.array(y_class_num)\n    # corresponding labels and corresponding gt bboxes\n    Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)\n\n    return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0), IoUs","execution_count":null,"outputs":[]},{"metadata":{"id":"RzdjjEFGQXnz","trusted":true},"cell_type":"code","source":"def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):\n\t\"\"\"Convert rpn layer to roi bboxes\n\n\tArgs: (num_anchors = 9)\n\t\trpn_layer: output layer for rpn classification \n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 18) if resized image is 400 width and 300\n\t\tregr_layer: output layer for rpn regression\n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 72) if resized image is 400 width and 300\n\t\tC: config\n\t\tuse_regr: Wether to use bboxes regression in rpn\n\t\tmax_boxes: max bboxes number for non-max-suppression (NMS)\n\t\toverlap_thresh: If iou in NMS is larger than this threshold, drop the box\n\n\tReturns:\n\t\tresult: boxes from non-max-suppression (shape=(300, 4))\n\t\t\tboxes: coordinates for bboxes (on the feature map)\n\t\"\"\"\n\tregr_layer = regr_layer / C.std_scaling\n\n\tanchor_sizes = C.anchor_box_scales   # (3 in here)\n\tanchor_ratios = C.anchor_box_ratios  # (3 in here)\n\n\tassert rpn_layer.shape[0] == 1\n\n\t(rows, cols) = rpn_layer.shape[1:3]\n\n\tcurr_layer = 0\n\n\t# A.shape = (4, feature_map.height, feature_map.width, num_anchors) \n\t# Might be (4, 18, 25, 18) if resized image is 400 width and 300\n\t# A is the coordinates for 9 anchors for every point in the feature map \n\t# => all 18x25x9=4050 anchors cooridnates\n\tA = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))\n\n\tfor anchor_size in anchor_sizes:\n\t\tfor anchor_ratio in anchor_ratios:\n\t\t\t# anchor_x = (128 * 1) / 16 = 8  => width of current anchor\n\t\t\t# anchor_y = (128 * 2) / 16 = 16 => height of current anchor\n\t\t\tanchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride\n\t\t\tanchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride\n\t\t\t\n\t\t\t# curr_layer: 0~8 (9 anchors)\n\t\t\t# the Kth anchor of all position in the feature map (9th in total)\n\t\t\tregr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4] # shape => (18, 25, 4)\n\t\t\tregr = np.transpose(regr, (2, 0, 1)) # shape => (4, 18, 25)\n\n\t\t\t# Create 18x25 mesh grid\n\t\t\t# For every point in x, there are all the y points and vice versa\n\t\t\t# X.shape = (18, 25)\n\t\t\t# Y.shape = (18, 25)\n\t\t\tX, Y = np.meshgrid(np.arange(cols),np. arange(rows))\n\n\t\t\t# Calculate anchor position and size for each feature map point\n\t\t\tA[0, :, :, curr_layer] = X - anchor_x/2 # Top left x coordinate\n\t\t\tA[1, :, :, curr_layer] = Y - anchor_y/2 # Top left y coordinate\n\t\t\tA[2, :, :, curr_layer] = anchor_x       # width of current anchor\n\t\t\tA[3, :, :, curr_layer] = anchor_y       # height of current anchor\n\n\t\t\t# Apply regression to x, y, w and h if there is rpn regression layer\n\t\t\tif use_regr:\n\t\t\t\tA[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)\n\n\t\t\t# Avoid width and height exceeding 1\n\t\t\tA[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])\n\n\t\t\t# Convert (x, y , w, h) to (x1, y1, x2, y2)\n\t\t\t# x1, y1 is top left coordinate\n\t\t\t# x2, y2 is bottom right coordinate\n\t\t\tA[2, :, :, curr_layer] += A[0, :, :, curr_layer]\n\t\t\tA[3, :, :, curr_layer] += A[1, :, :, curr_layer]\n\n\t\t\t# Avoid bboxes drawn outside the feature map\n\t\t\tA[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])\n\t\t\tA[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])\n\t\t\tA[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])\n\n\t\t\tcurr_layer += 1\n\n\tall_boxes = np.reshape(A.transpose((0, 3, 1, 2)), (4, -1)).transpose((1, 0))  # shape=(4050, 4)\n\tall_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))                   # shape=(4050,)\n\n\tx1 = all_boxes[:, 0]\n\ty1 = all_boxes[:, 1]\n\tx2 = all_boxes[:, 2]\n\ty2 = all_boxes[:, 3]\n\n\t# Find out the bboxes which is illegal and delete them from bboxes list\n\tidxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))\n\n\tall_boxes = np.delete(all_boxes, idxs, 0)\n\tall_probs = np.delete(all_probs, idxs, 0)\n\n\t# Apply non_max_suppression\n\t# Only extract the bboxes. Don't need rpn probs in the later process\n\tresult = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]\n\n\treturn result","execution_count":null,"outputs":[]},{"metadata":{"id":"H7dnbkcwQphW","outputId":"107c685a-14b9-4c04-b71e-471821542702","trusted":true},"cell_type":"code","source":"p = os.getcwd()\np","execution_count":null,"outputs":[]},{"metadata":{"id":"SLludeeuQeYx","trusted":true},"cell_type":"code","source":"base_path = '/kaggle/input'\n\ntrain_path =  '/kaggle/input/processed/annotation.txt' # Training data (annotation file)\n\nnum_rois = 4 # Number of RoIs to process at once.\n\n# Augmentation flag\nhorizontal_flips = True # Augment with horizontal flips in training. \nvertical_flips = True   # Augment with vertical flips in training. \nrot_90 = True           # Augment with 90 degree rotations in training. \n\noutput_weight_path = os.path.join('model_frcnn_vgg.hdf5')\n\nrecord_path = os.path.join('record.csv') # Record data (used to save the losses, classification accuracy and mean average precision)\n\nbase_weight_path = os.path.join(base_path, 'models/model/vgg16_weights_tf_dim_ordering_tf_kernels.h5')\n\nconfig_output_filename = os.path.join('model_vgg_config.pickle')","execution_count":null,"outputs":[]},{"metadata":{"id":"dqBrd6OqRrzQ","trusted":true},"cell_type":"code","source":"# Create the config\nC = Config()\n\nC.use_horizontal_flips = horizontal_flips\nC.use_vertical_flips = vertical_flips\nC.rot_90 = rot_90\n\nC.record_path = record_path\nC.model_path = output_weight_path\nC.num_rois = num_rois\n\nC.base_net_weights = base_weight_path","execution_count":null,"outputs":[]},{"metadata":{"id":"P-JWc3Cg4nt1","outputId":"d9bf6b59-c214-4ab0-89f7-65da13dec50c","trusted":true},"cell_type":"code","source":"C.base_net_weights","execution_count":null,"outputs":[]},{"metadata":{"id":"pQ4rzBO4RsiD","outputId":"09a3cf7b-18a7-442b-99fc-56e4cd5a232b","trusted":true},"cell_type":"code","source":"#--------------------------------------------------------#\n# This step will spend some time to load the data        #\n#--------------------------------------------------------#\nst = time.time()\ntrain_imgs, classes_count, class_mapping = get_data(train_path)\nprint()\nprint('Spend %0.2f mins to load the data' % ((time.time()-st)/60) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_mapping","execution_count":null,"outputs":[]},{"metadata":{"id":"P0GIxRgORwlu","outputId":"b77c86c2-0a93-4759-ac4e-096dd61c7ffc","trusted":true},"cell_type":"code","source":"if 'bg' not in classes_count:\n\tclasses_count['bg'] = 0\n\tclass_mapping['bg'] = len(class_mapping)\n# e.g.\n#    classes_count: {'usask_1': 5807,'arvalis_1': 45716,'inrae_1': 3701,'ethz_1': 51489,'arvalis_3': 16665,'rres_1': 9635,'bg':0}\n#    class_mapping: {'usask_1': 0,'arvalis_1': 1,'inrae_1': 2,'ethz_1': 3,'arvalis_3': 4,'rres_1': 5,'bg': 6}\nC.class_mapping = class_mapping\n\nprint('Training images per class:')\npprint.pprint(classes_count)\nprint('Num classes (including bg) = {}'.format(len(classes_count)))\nprint(class_mapping)\n\n# Save the configuration\nwith open(config_output_filename, 'wb') as config_f:\n\tpickle.dump(C,config_f)\n\tprint('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename))\n","execution_count":null,"outputs":[]},{"metadata":{"id":"vIRZoHCZrFPl","outputId":"4f640b07-c213-4b06-cc5b-f96b8471a916","trusted":true},"cell_type":"code","source":"# Shuffle the images with seed\nrandom.seed(1)\nrandom.shuffle(train_imgs)\n\nprint('Num train samples (images) {}'.format(len(train_imgs)))","execution_count":null,"outputs":[]},{"metadata":{"id":"lpKVm6D1rTb6","trusted":true},"cell_type":"code","source":"# Get train data generator which generate X, Y, image_data\ndata_gen_train = get_anchor_gt(train_imgs, C, get_img_output_length, mode='train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"id":"ymkExpDJrjqu"},"cell_type":"markdown","source":"#### Explore 'data_gen_train'\n\ndata_gen_train is an **generator**, so we get the data by calling **next(data_gen_train)**","execution_count":null},{"metadata":{"id":"IrFf1aW0rcty","trusted":true},"cell_type":"code","source":"X, Y, image_data, debug_img, debug_num_pos = next(data_gen_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"debug_num_pos","execution_count":null,"outputs":[]},{"metadata":{"id":"AJjqYGcSroN2","outputId":"e8b3cf58-327b-4c07-8efc-1fb655ff8298","trusted":true},"cell_type":"code","source":"print('Original image: height=%d width=%d'%(image_data['height'], image_data['width']))\nprint('Resized image:  height=%d width=%d C.im_size=%d'%(X.shape[1], X.shape[2], C.im_size))\nprint('Feature map size: height=%d width=%d C.rpn_stride=%d'%(Y[0].shape[1], Y[0].shape[2], C.rpn_stride))\nprint(X.shape)\nprint(str(len(Y))+\" includes 'y_rpn_cls' and 'y_rpn_regr'\")\nprint('Shape of y_rpn_cls {}'.format(Y[0].shape))\nprint('Shape of y_rpn_regr {}'.format(Y[1].shape))\nprint(image_data)\n\nprint('Number of positive anchors for this image: %d' % (debug_num_pos))\nif debug_num_pos==0:\n    gt_x1, gt_x2 = image_data['bboxes'][0]['x1']*(X.shape[2]/image_data['height']), image_data['bboxes'][0]['x2']*(X.shape[2]/image_data['height'])\n    gt_y1, gt_y2 = image_data['bboxes'][0]['y1']*(X.shape[1]/image_data['width']), image_data['bboxes'][0]['y2']*(X.shape[1]/image_data['width'])\n    gt_x1, gt_y1, gt_x2, gt_y2 = int(gt_x1), int(gt_y1), int(gt_x2), int(gt_y2)\n\n    img = debug_img.copy()\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    color = (0, 255, 0)\n    cv2.putText(img, 'gt bbox', (gt_x1, gt_y1-5), cv2.FONT_HERSHEY_DUPLEX, 0.7, color, 1)\n    cv2.rectangle(img, (gt_x1, gt_y1), (gt_x2, gt_y2), color, 2)\n    cv2.circle(img, (int((gt_x1+gt_x2)/2), int((gt_y1+gt_y2)/2)), 3, color, -1)\n\n    plt.grid()\n    plt.imshow(img)\n    plt.show()\nelse:\n    cls = Y[0][0]\n    pos_cls = np.where(cls==1)\n    print(pos_cls)\n    regr = Y[1][0]\n    pos_regr = np.where(regr==1)\n    print(pos_regr)\n    print('y_rpn_cls for possible pos anchor: {}'.format(cls[pos_cls[0][0],pos_cls[1][0],:]))\n    print('y_rpn_regr for positive anchor: {}'.format(regr[pos_regr[0][0],pos_regr[1][0],:]))\n\n    gt_x1, gt_x2 = image_data['bboxes'][0]['x1']*(X.shape[2]/image_data['width']), image_data['bboxes'][0]['x2']*(X.shape[2]/image_data['width'])\n    gt_y1, gt_y2 = image_data['bboxes'][0]['y1']*(X.shape[1]/image_data['height']), image_data['bboxes'][0]['y2']*(X.shape[1]/image_data['height'])\n    gt_x1, gt_y1, gt_x2, gt_y2 = int(gt_x1), int(gt_y1), int(gt_x2), int(gt_y2)\n\n    img = debug_img.copy()\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    color = (0, 255, 0)\n    #   cv2.putText(img, 'gt bbox', (gt_x1, gt_y1-5), cv2.FONT_HERSHEY_DUPLEX, 0.7, color, 1)\n    cv2.rectangle(img, (gt_x1, gt_y1), (gt_x2, gt_y2), color, 2)\n    cv2.circle(img, (int((gt_x1+gt_x2)/2), int((gt_y1+gt_y2)/2)), 3, color, -1)\n\n    # Add text\n    textLabel = 'gt bbox'\n    (retval,baseLine) = cv2.getTextSize(textLabel,cv2.FONT_HERSHEY_COMPLEX,0.5,1)\n    textOrg = (gt_x1, gt_y1+5)\n    cv2.rectangle(img, (textOrg[0] - 5, textOrg[1]+baseLine - 5), (textOrg[0]+retval[0] + 5, textOrg[1]-retval[1] - 5), (0, 0, 0), 2)\n    cv2.rectangle(img, (textOrg[0] - 5,textOrg[1]+baseLine - 5), (textOrg[0]+retval[0] + 5, textOrg[1]-retval[1] - 5), (255, 255, 255), -1)\n    cv2.putText(img, textLabel, textOrg, cv2.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 0), 1)\n\n    # Draw positive anchors according to the y_rpn_regr\n    for i in range(debug_num_pos):\n\n        color = (100+i*(155/4), 0, 100+i*(155/4))\n\n        idx = pos_regr[2][i*4]/4\n        anchor_size = C.anchor_box_scales[int(idx/3)]\n        anchor_ratio = C.anchor_box_ratios[2-int((idx+1)%3)]\n\n        center = (pos_regr[1][i*4]*C.rpn_stride, pos_regr[0][i*4]*C.rpn_stride)\n        print('Center position of positive anchor: ', center)\n        cv2.circle(img, center, 3, color, -1)\n        anc_w, anc_h = anchor_size*anchor_ratio[0], anchor_size*anchor_ratio[1]\n        cv2.rectangle(img, (center[0]-int(anc_w/2), center[1]-int(anc_h/2)), (center[0]+int(anc_w/2), center[1]+int(anc_h/2)), color, 2)\n#         cv2.putText(img, 'pos anchor bbox '+str(i+1), (center[0]-int(anc_w/2), center[1]-int(anc_h/2)-5), cv2.FONT_HERSHEY_DUPLEX, 0.5, color, 1)\n\nprint('Green bboxes is ground-truth bbox. Others are positive anchors')\nplt.figure(figsize=(8,8))\nplt.grid()\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"NiDLPIkkr3I9"},"cell_type":"markdown","source":"Build the Model","execution_count":null},{"metadata":{"id":"be75GVAEruRm","trusted":true},"cell_type":"code","source":"input_shape_img = (None, None, 3)\n\nimg_input = Input(shape=input_shape_img)\nroi_input = Input(shape=(None, 4))\n\n# define the base network (VGG here, can be Resnet50, Inception, etc)\nshared_layers = nn_base(img_input, trainable=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"da63dP-csIIz","outputId":"7db44b46-4d65-4a66-cc18-0246d588f399","trusted":true},"cell_type":"code","source":"# define the RPN, built on the base layers\nnum_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) # 9\nrpn = rpn_layer(shared_layers, num_anchors)\n\nclassifier = classifier_layer(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count))\n\nmodel_rpn = Model(img_input, rpn[:2])\nmodel_classifier = Model([img_input, roi_input], classifier)\n\n# this is a model that holds both the RPN and the classifier, used to load/save weights for the models\nmodel_all = Model([img_input, roi_input], rpn[:2] + classifier)\n\n# Because the google colab can only run the session several hours one time (then you need to connect again), \n# we need to save the model and load the model to continue training\nif not os.path.isfile(C.model_path):\n    #If this is the begin of the training, load the pre-traind base network such as vgg-16\n    try:\n        print('This is the first time of your training')\n        print('loading weights from {}'.format(C.base_net_weights))\n        model_rpn.load_weights(C.base_net_weights, by_name=True)\n        model_classifier.load_weights(C.base_net_weights, by_name=True)\n    except:\n        print('Could not load pretrained model weights. Weights can be found in the keras application folder \\\n            https://github.com/fchollet/keras/tree/master/keras/applications')\n    \n    # Create the record.csv file to record losses, acc and mAP\n    record_df = pd.DataFrame(columns=['mean_overlapping_bboxes', 'class_acc', 'loss_rpn_cls', 'loss_rpn_regr', 'loss_class_cls', 'loss_class_regr', 'curr_loss', 'elapsed_time', 'mAP'])\nelse:\n    # If this is a continued training, load the trained model from before\n    print('Continue training based on previous trained model')\n    print('Loading weights from {}'.format(C.model_path))\n    model_rpn.load_weights(C.model_path, by_name=True)\n    model_classifier.load_weights(C.model_path, by_name=True)\n    \n    # Load the records\n    record_df = pd.read_csv(record_path)\n\n    r_mean_overlapping_bboxes = record_df['mean_overlapping_bboxes']\n    r_class_acc = record_df['class_acc']\n    r_loss_rpn_cls = record_df['loss_rpn_cls']\n    r_loss_rpn_regr = record_df['loss_rpn_regr']\n    r_loss_class_cls = record_df['loss_class_cls']\n    r_loss_class_regr = record_df['loss_class_regr']\n    r_curr_loss = record_df['curr_loss']\n    r_elapsed_time = record_df['elapsed_time']\n    r_mAP = record_df['mAP']\n\n    print('Already train %dK batches'% (len(record_df)))","execution_count":null,"outputs":[]},{"metadata":{"id":"KyW8RrCksOFa","trusted":true},"cell_type":"code","source":"optimizer = Adam(lr=1e-5)\noptimizer_classifier = Adam(lr=1e-5)\nmodel_rpn.compile(optimizer=optimizer, loss=[rpn_loss_cls(num_anchors), rpn_loss_regr(num_anchors)])\nmodel_classifier.compile(optimizer=optimizer_classifier, loss=[class_loss_cls, class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})\nmodel_all.compile(optimizer='sgd', loss='mae')","execution_count":null,"outputs":[]},{"metadata":{"id":"xgfKMiQHBcoA","trusted":true},"cell_type":"code","source":"# Training setting\ntotal_epochs = len(record_df)\nr_epochs = len(record_df)\n\nepoch_length = 1000\nnum_epochs = 2   #Just of sharing the karnel running with 2 epoch , you try with min 20 epochs\niter_num = 0\n\ntotal_epochs += num_epochs\n\nlosses = np.zeros((epoch_length, 5))\nrpn_accuracy_rpn_monitor = []\nrpn_accuracy_for_epoch = []\n\nif len(record_df)==0:\n    best_loss = np.Inf\nelse:\n    best_loss = np.min(r_curr_loss)","execution_count":null,"outputs":[]},{"metadata":{"id":"-Wq5yfXVBjYR","outputId":"637126d2-f058-47da-d8ac-7d104806837d","trusted":true},"cell_type":"code","source":"print(len(record_df))","execution_count":null,"outputs":[]},{"metadata":{"id":"EYQ3tEolBmdl","outputId":"b14ae236-52df-4da9-ce0f-e6ff5b96c37a","trusted":true},"cell_type":"code","source":"start_time = time.time()\nfor epoch_num in range(num_epochs):\n\n    progbar = generic_utils.Progbar(epoch_length)\n    print('Epoch {}/{}'.format(r_epochs + 1, total_epochs))\n    \n    r_epochs += 1\n\n    while True:\n        try:\n\n            if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:\n                mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor))/len(rpn_accuracy_rpn_monitor)\n                rpn_accuracy_rpn_monitor = []\n#                 print('Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'.format(mean_overlapping_bboxes, epoch_length))\n                if mean_overlapping_bboxes == 0:\n                    print('RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.')\n\n            # Generate X (x_img) and label Y ([y_rpn_cls, y_rpn_regr])\n            X, Y, img_data, debug_img, debug_num_pos = next(data_gen_train)\n\n            # Train rpn model and get loss value [_, loss_rpn_cls, loss_rpn_regr]\n            loss_rpn = model_rpn.train_on_batch(X, Y)\n\n            # Get predicted rpn from rpn model [rpn_cls, rpn_regr]\n            P_rpn = model_rpn.predict_on_batch(X)\n\n            # R: bboxes (shape=(300,4))\n            # Convert rpn layer to roi bboxes\n            R = rpn_to_roi(P_rpn[0], P_rpn[1], C, K.common.image_dim_ordering(), use_regr=True, overlap_thresh=0.7, max_boxes=300)\n            \n            # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format\n            # X2: bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n            # Y1: one hot code for bboxes from above => x_roi (X)\n            # Y2: corresponding labels and corresponding gt bboxes\n            X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping)\n\n            # If X2 is None means there are no matching bboxes\n            if X2 is None:\n                rpn_accuracy_rpn_monitor.append(0)\n                rpn_accuracy_for_epoch.append(0)\n                continue\n            \n            # Find out the positive anchors and negative anchors\n            neg_samples = np.where(Y1[0, :, -1] == 1)\n            pos_samples = np.where(Y1[0, :, -1] == 0)\n\n            if len(neg_samples) > 0:\n                neg_samples = neg_samples[0]\n            else:\n                neg_samples = []\n\n            if len(pos_samples) > 0:\n                pos_samples = pos_samples[0]\n            else:\n                pos_samples = []\n\n            rpn_accuracy_rpn_monitor.append(len(pos_samples))\n            rpn_accuracy_for_epoch.append((len(pos_samples)))\n\n            if C.num_rois > 1:\n                # If number of positive anchors is larger than 4//2 = 2, randomly choose 2 pos samples\n                if len(pos_samples) < C.num_rois//2:\n                    selected_pos_samples = pos_samples.tolist()\n                else:\n                    selected_pos_samples = np.random.choice(pos_samples, C.num_rois//2, replace=False).tolist()\n                \n                # Randomly choose (num_rois - num_pos) neg samples\n                try:\n                    selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist()\n                except:\n                    selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist()\n                \n                # Save all the pos and neg samples in sel_samples\n                sel_samples = selected_pos_samples + selected_neg_samples\n            else:\n                # in the extreme case where num_rois = 1, we pick a random pos or neg sample\n                selected_pos_samples = pos_samples.tolist()\n                selected_neg_samples = neg_samples.tolist()\n                if np.random.randint(0, 2):\n                    sel_samples = random.choice(neg_samples)\n                else:\n                    sel_samples = random.choice(pos_samples)\n\n            # training_data: [X, X2[:, sel_samples, :]]\n            # labels: [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]\n            #  X                     => img_data resized image\n            #  X2[:, sel_samples, :] => num_rois (4 in here) bboxes which contains selected neg and pos\n            #  Y1[:, sel_samples, :] => one hot encode for num_rois bboxes which contains selected neg and pos\n            #  Y2[:, sel_samples, :] => labels and gt bboxes for num_rois bboxes which contains selected neg and pos\n            loss_class = model_classifier.train_on_batch([X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]])\n\n            losses[iter_num, 0] = loss_rpn[1]\n            losses[iter_num, 1] = loss_rpn[2]\n\n            losses[iter_num, 2] = loss_class[1]\n            losses[iter_num, 3] = loss_class[2]\n            losses[iter_num, 4] = loss_class[3]\n\n            iter_num += 1\n\n            progbar.update(iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])),\n                                      ('final_cls', np.mean(losses[:iter_num, 2])), ('final_regr', np.mean(losses[:iter_num, 3]))])\n\n            if iter_num == epoch_length:\n                loss_rpn_cls = np.mean(losses[:, 0])\n                loss_rpn_regr = np.mean(losses[:, 1])\n                loss_class_cls = np.mean(losses[:, 2])\n                loss_class_regr = np.mean(losses[:, 3])\n                class_acc = np.mean(losses[:, 4])\n\n                mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)\n                rpn_accuracy_for_epoch = []\n\n                if C.verbose:\n                    print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes))\n                    print('Classifier accuracy for bounding boxes from RPN: {}'.format(class_acc))\n                    print('Loss RPN classifier: {}'.format(loss_rpn_cls))\n                    print('Loss RPN regression: {}'.format(loss_rpn_regr))\n                    print('Loss Detector classifier: {}'.format(loss_class_cls))\n                    print('Loss Detector regression: {}'.format(loss_class_regr))\n                    print('Total loss: {}'.format(loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr))\n                    print('Elapsed time: {}'.format(time.time() - start_time))\n                    elapsed_time = (time.time()-start_time)/60\n\n                curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr\n                iter_num = 0\n                start_time = time.time()\n\n                if curr_loss < best_loss:\n                    if C.verbose:\n                        print('Total loss decreased from {} to {}, saving weights'.format(best_loss,curr_loss))\n                    best_loss = curr_loss\n                    model_all.save_weights(C.model_path)\n\n                new_row = {'mean_overlapping_bboxes':round(mean_overlapping_bboxes, 3), \n                           'class_acc':round(class_acc, 3), \n                           'loss_rpn_cls':round(loss_rpn_cls, 3), \n                           'loss_rpn_regr':round(loss_rpn_regr, 3), \n                           'loss_class_cls':round(loss_class_cls, 3), \n                           'loss_class_regr':round(loss_class_regr, 3), \n                           'curr_loss':round(curr_loss, 3), \n                           'elapsed_time':round(elapsed_time, 3), \n                           'mAP': 0}\n\n                record_df = record_df.append(new_row, ignore_index=True)\n                record_df.to_csv(record_path, index=0)\n\n                break\n\n        except Exception as e:\n            print('Exception: {}'.format(e))\n            continue\n\nprint('Training complete, exiting.')","execution_count":null,"outputs":[]},{"metadata":{"id":"oMlHi6fxx7BD","outputId":"2446cc3d-9dd5-44a4-a476-f6f6f60d6da0","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['mean_overlapping_bboxes'], 'r')\nplt.title('mean_overlapping_bboxes')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['class_acc'], 'r')\nplt.title('class_acc')\n\nplt.show()\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['loss_rpn_cls'], 'r')\nplt.title('loss_rpn_cls')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['loss_rpn_regr'], 'r')\nplt.title('loss_rpn_regr')\nplt.show()\n\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['loss_class_cls'], 'r')\nplt.title('loss_class_cls')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['loss_class_regr'], 'r')\nplt.title('loss_class_regr')\nplt.show()\n\nplt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'r')\nplt.title('total_loss')\nplt.show()\n\n# plt.figure(figsize=(15,5))\n# plt.subplot(1,2,1)\n# plt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'r')\n# plt.title('total_loss')\n# plt.subplot(1,2,2)\n# plt.plot(np.arange(0, r_epochs), record_df['elapsed_time'], 'r')\n# plt.title('elapsed_time')\n# plt.show()\n\n# plt.title('loss')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_rpn_cls'], 'b')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_rpn_regr'], 'g')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_class_cls'], 'r')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_class_regr'], 'c')\n# # plt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'm')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"2bgxOldTx6zn"},"cell_type":"markdown","source":"Part 1: https://www.kaggle.com/kishor1210/eda-and-data-processing\n\nPart 2: https://www.kaggle.com/kishor1210/train-faster-rcnn-using-keras\n\nPart 3: comming soon....","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}