{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/input/faster-rcnn')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keras.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nsrc = '../input/faster-rcnn/annotation.txt'\ndst = '/kaggle/working/annotation.txt'\nd = shutil.copy(src,dst)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import gdown\n# url = 'https://drive.google.com/uc?id=1IgxPP0aI5pxyPHVSM2ZJjN1p9dtE4_64&export=download'\n# output = '/kaggle/working/rpn_vgg.hdf5'\n# gdown.download(url, output, quiet=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import pickle\n# infile = open('/kaggle/working/config.pickle','rb')\n# config_file = pickle.load(infile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import division\nimport random\nimport pprint\nimport sys\nimport time\nimport numpy as np\nfrom optparse import OptionParser\nimport pickle\nimport copy\nfrom keras import backend as K\nfrom keras.optimizers import Adam, SGD, RMSprop\n# from keras.layers import Input\nfrom keras.models import Model\n# from keras_frcnn import config, data_generators\n# from keras_frcnn import losses as losses\n# import keras_frcnn.roi_helpers as roi_helpers\nfrom keras.utils import generic_utils\n\nsys.setrecursionlimit(40000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/voctrain/VOCdevkit/VOC2007')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '/kaggle/input/faster-rcnn/annotation.txt'\nparser = 'simple'\nnum_rois = 32\nnetwork = 'vgg'\nhorizontal_flips = False\nvertical_flips = False\nrot_90 = False\nnum_epochs = 2000\nconfig_filename = '/kaggle/working/config.pickle'\noutput_weight_path = '/kaggle/working/model_frcnn.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nimport math\n\nclass Config:\n\n\tdef __init__(self):\n\n\t\tself.verbose = True\n\n\t\tself.network = 'resnet50'\n\n\t\t# setting for data augmentation\n\t\tself.use_horizontal_flips = False\n\t\tself.use_vertical_flips = False\n\t\tself.rot_90 = False\n\n\t\t# anchor box scales\n\t\tself.anchor_box_scales = [128, 256, 512]\n\n\t\t# anchor box ratios\n\t\tself.anchor_box_ratios = [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]\n\n\t\t# size to resize the smallest side of the image\n\t\tself.im_size = 600\n\n\t\t# image channel-wise mean to subtract\n\t\tself.img_channel_mean = [103.939, 116.779, 123.68]\n\t\tself.img_scaling_factor = 1.0\n\n\t\t# number of ROIs at once\n\t\tself.num_rois = 4\n\n\t\t# stride at the RPN (this depends on the network configuration)\n\t\tself.rpn_stride = 16\n\n\t\tself.balanced_classes = False\n\n\t\t# scaling the stdev\n\t\tself.std_scaling = 4.0\n\t\tself.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]\n\n\t\t# overlaps for RPN\n\t\tself.rpn_min_overlap = 0.3\n\t\tself.rpn_max_overlap = 0.7\n\n\t\t# overlaps for classifier ROIs\n\t\tself.classifier_min_overlap = 0.1\n\t\tself.classifier_max_overlap = 0.5\n\n\t\t# placeholder for the class mapping, automatically generated by the parser\n\t\tself.class_mapping = None\n\n\t\t#location of pretrained weights for the base network \n\t\t# weight files can be found at:\n\t\t# https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels_notop.h5\n\t\t# https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\n\n\t\tself.model_path = '/kaggle/input/faster-rcnn/model_frcnn_25.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_weight_path():\n    if K.image_dim_ordering() == 'th':\n        print('pretrained weights not available for VGG with theano backend')\n        return\n    else:\n        return '/kaggle/input/firstrun/model_frcnn.hdf5'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C = Config()\n\nC.use_horizontal_flips = bool(horizontal_flips)\nC.use_vertical_flips = bool(vertical_flips)\nC.rot_90 = bool(rot_90)\n\nC.model_path = output_weight_path\nC.num_rois = int(num_rois)\n\nnetwork == 'vgg'\nC.network = 'vgg'\n# \tfrom keras_frcnn import vgg as nn\n# network == 'resnet50'\n# # \tfrom keras_frcnn import resnet as nn\n# C.network = 'resnet50'\n# else:\n# \tprint('Not a valid model')\n# \traise ValueError\n\ninput_weight_path = None\n# check if weight path was passed via command line\nif input_weight_path is None:\n\tC.base_net_weights = input_weight_path\nelse:\n\t# set the path to weights based on backend and model\n\tC.base_net_weights = get_weight_path()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport numpy as np\n\ndef get_data(input_path):\n\tfound_bg = False\n\tall_imgs = {}\n\n\tclasses_count = {}\n\n\tclass_mapping = {}\n\n\tvisualise = True\n\t\n\twith open(input_path,'r') as f:\n\n\t\tprint('Parsing annotation files')\n\n\t\tfor line in f:\n\t\t\tline_split = line.strip().split(',')\n\t\t\t(filename,x1,y1,x2,y2,class_name) = line_split\n\n\t\t\tif class_name not in classes_count:\n\t\t\t\tclasses_count[class_name] = 1\n\t\t\telse:\n\t\t\t\tclasses_count[class_name] += 1\n\n\t\t\tif class_name not in class_mapping:\n\t\t\t\tif class_name == 'bg' and found_bg == False:\n\t\t\t\t\tprint('Found class name with special name bg. Will be treated as a background region (this is usually for hard negative mining).')\n\t\t\t\t\tfound_bg = True\n\t\t\t\tclass_mapping[class_name] = len(class_mapping)\n\n\t\t\tif filename not in all_imgs:\n\t\t\t\tall_imgs[filename] = {}\n\t\t\t\t\n\t\t\t\timg = cv2.imread(filename)\n\t\t\t\t(rows,cols) = img.shape[:2]\n\t\t\t\tall_imgs[filename]['filepath'] = filename\n\t\t\t\tall_imgs[filename]['width'] = cols\n\t\t\t\tall_imgs[filename]['height'] = rows\n\t\t\t\tall_imgs[filename]['bboxes'] = []\n\t\t\t\tif np.random.randint(0,6) > 0:\n\t\t\t\t\tall_imgs[filename]['imageset'] = 'trainval'\n\t\t\t\telse:\n\t\t\t\t\tall_imgs[filename]['imageset'] = 'test'\n\n\t\t\tall_imgs[filename]['bboxes'].append({'class': class_name, 'x1': int(x1), 'x2': int(x2), 'y1': int(y1), 'y2': int(y2)})\n\n\n\t\tall_data = []\n\t\tfor key in all_imgs:\n\t\t\tall_data.append(all_imgs[key])\n\t\t\n\t\t# make sure the bg class is last in the list\n\t\tif found_bg:\n\t\t\tif class_mapping['bg'] != len(class_mapping) - 1:\n\t\t\t\tkey_to_switch = [key for key in class_mapping.keys() if class_mapping[key] == len(class_mapping)-1][0]\n\t\t\t\tval_to_switch = class_mapping['bg']\n\t\t\t\tclass_mapping['bg'] = len(class_mapping) - 1\n\t\t\t\tclass_mapping[key_to_switch] = val_to_switch\n\t\t\n\t\treturn all_data, classes_count, class_mapping\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_imgs, classes_count, class_mapping = get_data(train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_mapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nif 'bg' not in classes_count:\n\tclasses_count['bg'] = 0\n\tclass_mapping['bg'] = len(class_mapping)\n\nC.class_mapping = class_mapping\n\ninv_map = {v: k for k, v in class_mapping.items()}\n\nprint('Training images per class:')\npprint.pprint(classes_count)\nprint('Num classes (including bg) = {}'.format(len(classes_count)))\n\nconfig_output_filename = config_filename\n\nwith open(config_output_filename, 'wb') as config_f:\n\tpickle.dump(C,config_f)\n\tprint('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename))\n\nrandom.shuffle(all_imgs)\n\nnum_imgs = len(all_imgs)\n\ntrain_imgs = [s for s in all_imgs if s['imageset'] == 'trainval']\nval_imgs = [s for s in all_imgs if s['imageset'] == 'test']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras.engine import Layer, InputSpec\n# from keras import initializers, regularizers\n# from keras import backend as K\n\n\n# class FixedBatchNormalization(Layer):\n\n#     def __init__(self, epsilon=1e-3, axis=-1,\n#                  weights=None, beta_init='zero', gamma_init='one',\n#                  gamma_regularizer=None, beta_regularizer=None, **kwargs):\n\n#         self.supports_masking = True\n#         self.beta_init = initializers.get(beta_init)\n#         self.gamma_init = initializers.get(gamma_init)\n#         self.epsilon = epsilon\n#         self.axis = axis\n#         self.gamma_regularizer = regularizers.get(gamma_regularizer)\n#         self.beta_regularizer = regularizers.get(beta_regularizer)\n#         self.initial_weights = weights\n#         super(FixedBatchNormalization, self).__init__(**kwargs)\n\n#     def build(self, input_shape):\n#         self.input_spec = [InputSpec(shape=input_shape)]\n#         shape = (input_shape[self.axis],)\n\n#         self.gamma = self.add_weight(shape= shape,\n#                                      initializer=self.gamma_init,\n#                                      regularizer=self.gamma_regularizer,\n#                                      name='{}_gamma'.format(self.name),\n#                                      trainable=False)\n#         self.beta = self.add_weight(shape=shape,\n#                                     initializer=self.beta_init,\n#                                     regularizer=self.beta_regularizer,\n#                                     name='{}_beta'.format(self.name),\n#                                     trainable=False)\n#         self.running_mean = self.add_weight(shape=shape, initializer='zero',\n#                                             name='{}_running_mean'.format(self.name),\n#                                             trainable=False)\n#         self.running_std = self.add_weight(shape=shape, initializer='one',\n#                                            name='{}_running_std'.format(self.name),\n#                                            trainable=False)\n\n#         if self.initial_weights is not None:\n#             self.set_weights(self.initial_weights)\n#             del self.initial_weights\n\n#         self.built = True\n\n#     def call(self, x, mask=None):\n\n#         assert self.built, 'Layer must be built before being called'\n#         input_shape = K.int_shape(x)\n\n#         reduction_axes = list(range(len(input_shape)))\n#         del reduction_axes[self.axis]\n#         broadcast_shape = [1] * len(input_shape)\n#         broadcast_shape[self.axis] = input_shape[self.axis]\n\n#         if sorted(reduction_axes) == range(K.ndim(x))[:-1]:\n#             x_normed = K.batch_normalization(\n#                 x, self.running_mean, self.running_std,\n#                 self.beta, self.gamma,\n#                 epsilon=self.epsilon)\n#         else:\n#             # need broadcasting\n#             broadcast_running_mean = K.reshape(self.running_mean, broadcast_shape)\n#             broadcast_running_std = K.reshape(self.running_std, broadcast_shape)\n#             broadcast_beta = K.reshape(self.beta, broadcast_shape)\n#             broadcast_gamma = K.reshape(self.gamma, broadcast_shape)\n#             x_normed = K.batch_normalization(\n#                 x, broadcast_running_mean, broadcast_running_std,\n#                 broadcast_beta, broadcast_gamma,\n#                 epsilon=self.epsilon)\n\n#         return x_normed\n\n#     def get_config(self):\n#         config = {'epsilon': self.epsilon,\n#                   'axis': self.axis,\n#                   'gamma_regularizer': self.gamma_regularizer.get_config() if self.gamma_regularizer else None,\n#                   'beta_regularizer': self.beta_regularizer.get_config() if self.beta_regularizer else None}\n#         base_config = super(FixedBatchNormalization, self).get_config()\n#         return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers.core import Dense, Dropout, Flatten\nfrom keras.layers.convolutional import Conv2D, MaxPooling2D, SeparableConv2D","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras.layers import Input, Add, Activation, Convolution2D, ZeroPadding2D,AveragePooling2D, TimeDistributed\n\n# from keras import backend as K\n\n\n\n# def get_img_output_length(width, height):\n#     def get_output_length(input_length):\n#         # zero_pad\n#         input_length += 6\n#         # apply 4 strided convolutions\n#         filter_sizes = [7, 3, 1, 1]\n#         stride = 2\n#         for filter_size in filter_sizes:\n#             input_length = (input_length - filter_size + stride) // stride\n#         return input_length\n\n#     return get_output_length(width), get_output_length(height) \n\n# def identity_block(input_tensor, kernel_size, filters, stage, block, trainable=True):\n\n#     nb_filter1, nb_filter2, nb_filter3 = filters\n    \n#     bn_axis = 3\n#     conv_name_base = 'res' + str(stage) + block + '_branch'\n#     bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n#     x = Convolution2D(nb_filter1, (1, 1), name=conv_name_base + '2a', trainable=trainable)(input_tensor)\n#     x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)\n#     x = Activation('relu')(x)\n\n#     x = Convolution2D(nb_filter2, (kernel_size, kernel_size), padding='same', name=conv_name_base + '2b', trainable=trainable)(x)\n#     x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)\n#     x = Activation('relu')(x)\n\n#     x = Convolution2D(nb_filter3, (1, 1), name=conv_name_base + '2c', trainable=trainable)(x)\n#     x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)\n\n#     x = Add()([x, input_tensor])\n#     x = Activation('relu')(x)\n#     return x\n\n\n# def identity_block_td(input_tensor, kernel_size, filters, stage, block, trainable=True):\n\n#     # identity block time distributed\n\n#     nb_filter1, nb_filter2, nb_filter3 = filters\n# #     if K.image_dim_ordering() == 'tf':\n#     bn_axis = 3\n# #     else:\n# #         bn_axis = 1\n\n#     conv_name_base = 'res' + str(stage) + block + '_branch'\n#     bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n#     x = TimeDistributed(Convolution2D(nb_filter1, (1, 1), trainable=trainable, kernel_initializer='normal'), name=conv_name_base + '2a')(input_tensor)\n#     x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2a')(x)\n#     x = Activation('relu')(x)\n\n#     x = TimeDistributed(Convolution2D(nb_filter2, (kernel_size, kernel_size), trainable=trainable, kernel_initializer='normal',padding='same'), name=conv_name_base + '2b')(x)\n#     x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2b')(x)\n#     x = Activation('relu')(x)\n\n#     x = TimeDistributed(Convolution2D(nb_filter3, (1, 1), trainable=trainable, kernel_initializer='normal'), name=conv_name_base + '2c')(x)\n#     x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2c')(x)\n\n#     x = Add()([x, input_tensor])\n#     x = Activation('relu')(x)\n\n#     return x\n\n# def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2), trainable=True):\n\n#     nb_filter1, nb_filter2, nb_filter3 = filters\n# #     if K.image_dim_ordering() == 'tf':\n#     bn_axis = 3\n# #     else:\n# #         bn_axis = 1\n\n#     conv_name_base = 'res' + str(stage) + block + '_branch'\n#     bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n#     x = Convolution2D(nb_filter1, (1, 1), strides=strides, name=conv_name_base + '2a', trainable=trainable)(input_tensor)\n#     x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)\n#     x = Activation('relu')(x)\n\n#     x = Convolution2D(nb_filter2, (kernel_size, kernel_size), padding='same', name=conv_name_base + '2b', trainable=trainable)(x)\n#     x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)\n#     x = Activation('relu')(x)\n\n#     x = Convolution2D(nb_filter3, (1, 1), name=conv_name_base + '2c', trainable=trainable)(x)\n#     x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)\n\n#     shortcut = Convolution2D(nb_filter3, (1, 1), strides=strides, name=conv_name_base + '1', trainable=trainable)(input_tensor)\n#     shortcut = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)\n\n#     x = Add()([x, shortcut])\n#     x = Activation('relu')(x)\n#     return x\n\n\n# def conv_block_td(input_tensor, kernel_size, filters, stage, block, input_shape, strides=(2, 2), trainable=True):\n\n#     # conv block time distributed\n\n#     nb_filter1, nb_filter2, nb_filter3 = filters\n# #     if K.image_dim_ordering() == 'tf':\n#     bn_axis = 3\n# #     else:\n# #         bn_axis = 1\n\n#     conv_name_base = 'res' + str(stage) + block + '_branch'\n#     bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n#     x = TimeDistributed(Convolution2D(nb_filter1, (1, 1), strides=strides, trainable=trainable, kernel_initializer='normal'), input_shape=input_shape, name=conv_name_base + '2a')(input_tensor)\n#     x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2a')(x)\n#     x = Activation('relu')(x)\n\n#     x = TimeDistributed(Convolution2D(nb_filter2, (kernel_size, kernel_size), padding='same', trainable=trainable, kernel_initializer='normal'), name=conv_name_base + '2b')(x)\n#     x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2b')(x)\n#     x = Activation('relu')(x)\n\n#     x = TimeDistributed(Convolution2D(nb_filter3, (1, 1), kernel_initializer='normal'), name=conv_name_base + '2c', trainable=trainable)(x)\n#     x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2c')(x)\n\n#     shortcut = TimeDistributed(Convolution2D(nb_filter3, (1, 1), strides=strides, trainable=trainable, kernel_initializer='normal'), name=conv_name_base + '1')(input_tensor)\n#     shortcut = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '1')(shortcut)\n\n#     x = Add()([x, shortcut])\n#     x = Activation('relu')(x)\n#     return x\n\n# def nn_base(input_tensor=None, trainable=False):\n\n#     # Determine proper input shape\n# #     if K.image_dim_ordering() == 'th':\n# #     input_shape = (3, None, None)\n# #     else:\n#     input_shape = (None, None, 3)\n\n#     if input_tensor is None:\n#         img_input = Input(shape=input_shape)\n#     else:\n#         if not K.is_keras_tensor(input_tensor):\n#             img_input = Input(tensor=input_tensor, shape=input_shape)\n#         else:\n#             img_input = input_tensor\n\n# #     if K.image_dim_ordering() == 'tf':\n#     bn_axis = 3\n# #     else:\n# #         bn_axis = 1\n\n#     x = ZeroPadding2D((3, 3))(img_input)\n\n#     x = Convolution2D(64, (7, 7), strides=(2, 2), name='conv1', trainable = trainable)(x)\n#     x = FixedBatchNormalization(axis=bn_axis, name='bn_conv1')(x)\n#     x = Activation('relu')(x)\n#     x = MaxPooling2D((3, 3), strides=(2, 2))(x)\n\n#     x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), trainable = trainable)\n#     x = identity_block(x, 3, [64, 64, 256], stage=2, block='b', trainable = trainable)\n#     x = identity_block(x, 3, [64, 64, 256], stage=2, block='c', trainable = trainable)\n\n#     x = conv_block(x, 3, [128, 128, 512], stage=3, block='a', trainable = trainable)\n#     x = identity_block(x, 3, [128, 128, 512], stage=3, block='b', trainable = trainable)\n#     x = identity_block(x, 3, [128, 128, 512], stage=3, block='c', trainable = trainable)\n#     x = identity_block(x, 3, [128, 128, 512], stage=3, block='d', trainable = trainable)\n\n#     x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a', trainable = trainable)\n#     x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b', trainable = trainable)\n#     x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c', trainable = trainable)\n#     x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d', trainable = trainable)\n#     x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e', trainable = trainable)\n#     x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f', trainable = trainable)\n\n#     return x\n\n\n# def classifier_layers(x, input_shape, trainable=False):\n\n#     # compile times on theano tend to be very high, so we use smaller ROI pooling regions to workaround\n#     # (hence a smaller stride in the region that follows the ROI pool)\n# #     if K.backend() == 'tensorflow':\n#     x = conv_block_td(x, 3, [512, 512, 2048], stage=5, block='a', input_shape=input_shape, strides=(2, 2), trainable=trainable)\n# #     elif K.backend() == 'theano':\n# #         x = conv_block_td(x, 3, [512, 512, 2048], stage=5, block='a', input_shape=input_shape, strides=(1, 1), trainable=trainable)\n\n#     x = identity_block_td(x, 3, [512, 512, 2048], stage=5, block='b', trainable=trainable)\n#     x = identity_block_td(x, 3, [512, 512, 2048], stage=5, block='c', trainable=trainable)\n#     x = TimeDistributed(AveragePooling2D((7, 7)), name='avg_pool')(x)\n\n#     return x\n\n\n# def rpn(base_layers,num_anchors):\n\n#     x = Convolution2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)\n\n#     x_class = Convolution2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)\n#     x_regr = Convolution2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)\n\n#     return [x_class, x_regr, base_layers]\n\n# def classifier(base_layers, input_rois, num_rois, nb_classes = 21, trainable=False):\n\n#     # compile times on theano tend to be very high, so we use smaller ROI pooling regions to workaround\n\n# #     if K.backend() == 'tensorflow':\n#     pooling_regions = 14\n#     input_shape = (num_rois,14,14,1024)\n# #     elif K.backend() == 'theano':\n# #         pooling_regions = 7\n# #         input_shape = (num_rois,1024,7,7)\n\n#     out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois])\n#     out = classifier_layers(out_roi_pool, input_shape=input_shape, trainable=True)\n\n#     out = TimeDistributed(Flatten())(out)\n\n#     out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out)\n#     # note: no regression target for bg class\n#     out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out)\n#     return [out_class, out_regr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def nn_base(input_tensor=None, trainable=False):\n\n    input_shape = (None, None, 3)\n\n    if input_tensor is None:\n        img_input = Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n\n    bn_axis = 3\n\n\n    # Block 1\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)\n\n    # Block 2\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)\n\n    # Block 3\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv4')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)\n\n    # Block 4\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv4')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)\n\n    # Block 5\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)\n    \n    \n    # x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)\n\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function\nfrom __future__ import absolute_import\nfrom __future__ import division\n\nimport warnings\n\nfrom keras.models import Model\nfrom keras.layers import Input\nfrom keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, TimeDistributed\nfrom keras.engine.topology import get_source_inputs\nfrom keras.utils import layer_utils\nfrom keras.utils.data_utils import get_file\nfrom keras import backend as K\n# from keras_frcnn.RoiPoolingConv import RoiPoolingConv\n\n\ninput_shape_img = (None, None, 3)\nimg_input = Input(shape=input_shape_img)\nroi_input = Input(shape=(None, 4))\nshared_layers = nn_base(img_input, trainable=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn(base_layers, num_anchors):\n\n    x = Conv2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)\n\n    x_class = Conv2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)\n    x_regr = Conv2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)\n\n    return [x_class, x_regr, base_layers]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn = rpn(shared_layers, num_anchors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(rpn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.engine.topology import Layer\nimport keras.backend as K\n\nif K.backend() == 'tensorflow':\n    import tensorflow as tf\n\nclass RoiPoolingConv(Layer):     #Layer is parent class and RoiPoolingConv is subclass\n    def __init__(self, pool_size, num_rois, **kwargs):\n\n        self.dim_ordering = 'tf'\n        assert self.dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'\n\n        self.pool_size = pool_size\n        self.num_rois = num_rois\n\n        super(RoiPoolingConv, self).__init__(**kwargs) # Passing **kwargs to Layer class\n\n    def build(self, input_shape):\n        if self.dim_ordering == 'th':\n            self.nb_channels = input_shape[0][1]\n        elif self.dim_ordering == 'tf':\n            self.nb_channels = input_shape[0][3]\n\n    def compute_output_shape(self, input_shape):\n        if self.dim_ordering == 'th':\n            return None, self.num_rois, self.nb_channels, self.pool_size, self.pool_size\n        else:\n            return None, self.num_rois, self.pool_size, self.pool_size, self.nb_channels\n\n    def call(self, x, mask=None):\n\n        assert(len(x) == 2)\n\n        img = x[0]\n        rois = x[1]\n\n        input_shape = K.shape(img)\n\n        outputs = []\n\n        for roi_idx in range(self.num_rois):\n\n            x = rois[0, roi_idx, 0]\n            y = rois[0, roi_idx, 1]\n            w = rois[0, roi_idx, 2]\n            h = rois[0, roi_idx, 3]\n            \n            row_length = w / float(self.pool_size)\n            col_length = h / float(self.pool_size)\n\n            num_pool_regions = self.pool_size\n\n            #NOTE: the RoiPooling implementation differs between theano and tensorflow due to the lack of a resize op\n            # in theano. The theano implementation is much less efficient and leads to long compile times\n\n            if self.dim_ordering == 'th':\n                for jy in range(num_pool_regions):\n                    for ix in range(num_pool_regions):\n                        x1 = x + ix * row_length\n                        x2 = x1 + row_length\n                        y1 = y + jy * col_length\n                        y2 = y1 + col_length\n\n                        x1 = K.cast(x1, 'int32')\n                        x2 = K.cast(x2, 'int32')\n                        y1 = K.cast(y1, 'int32')\n                        y2 = K.cast(y2, 'int32')\n\n                        x2 = x1 + K.maximum(1,x2-x1)\n                        y2 = y1 + K.maximum(1,y2-y1)\n                        \n                        new_shape = [input_shape[0], input_shape[1],\n                                     y2 - y1, x2 - x1]\n\n                        x_crop = img[:, :, y1:y2, x1:x2]\n                        xm = K.reshape(x_crop, new_shape)\n                        pooled_val = K.max(xm, axis=(2, 3))\n                        outputs.append(pooled_val)\n\n            elif self.dim_ordering == 'tf':\n                x = K.cast(x, 'int32')\n                y = K.cast(y, 'int32')\n                w = K.cast(w, 'int32')\n                h = K.cast(h, 'int32')\n\n                rs = tf.image.resize(img[:, y:y+h, x:x+w, :], (self.pool_size, self.pool_size))\n                outputs.append(rs)\n\n        final_output = K.concatenate(outputs, axis=0)\n        final_output = K.reshape(final_output, (1, self.num_rois, self.pool_size, self.pool_size, self.nb_channels))\n\n        if self.dim_ordering == 'th':\n            final_output = K.permute_dimensions(final_output, (0, 1, 4, 2, 3))\n        else:\n            final_output = K.permute_dimensions(final_output, (0, 1, 2, 3, 4))\n\n        return final_output\n    \n    \n    def get_config(self):\n        config = {'pool_size': self.pool_size,\n                  'num_rois': self.num_rois}\n        base_config = super(RoiPoolingConv, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def classifier(base_layers, input_rois, num_rois, nb_classes = 21, trainable=False):\n\n    # compile times on theano tend to be very high, so we use smaller ROI pooling regions to workaround\n\n    if K.backend() == 'tensorflow':\n        pooling_regions = 7\n        input_shape = (num_rois,7,7,512)\n    elif K.backend() == 'theano':\n        pooling_regions = 7\n        input_shape = (num_rois,512,7,7)\n\n    out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois])\n    print (out_roi_pool)\n\n    out = TimeDistributed(Flatten(name='flatten'))(out_roi_pool)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc1'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc2'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n\n    out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out)\n    # note: no regression target for bg class\n    out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out)\n\n    return [out_class, out_regr]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier = classifier(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_rpn = Model(img_input, rpn[:2])\nmodel_classifier = Model([img_input, roi_input], classifier)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in model_rpn.layers:\n    print(layer.output_shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in model_classifier.layers:\n    print(layer.output_shape)\n#     print (layer.get_weights())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_all = Model([img_input, roi_input], rpn[:2] + classifier)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.objectives import categorical_crossentropy\n\nimport tensorflow as tf\n\nlambda_rpn_regr = 1.0\nlambda_rpn_class = 1.0\n\nlambda_cls_regr = 1.0\nlambda_cls_class = 1.0\n\nepsilon = 1e-4\n\n\ndef rpn_loss_regr(num_anchors):\n\tdef rpn_loss_regr_fixed_num(y_true, y_pred):\n\t\tx = y_true[:, :, :, 4 * num_anchors:] - y_pred\n\t\tx_abs = K.abs(x)\n\t\tx_bool = K.cast(K.less_equal(x_abs, 1.0), tf.float32)\n\t\treturn lambda_rpn_regr * K.sum(y_true[:, :, :, :4 * num_anchors] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :, :4 * num_anchors])\n\n\treturn rpn_loss_regr_fixed_num\n\n\ndef rpn_loss_cls(num_anchors):\n\tdef rpn_loss_cls_fixed_num(y_true, y_pred):\n\t\treturn lambda_rpn_class * K.sum(y_true[:, :, :, :num_anchors] * K.binary_crossentropy(y_pred[:, :, :, :], y_true[:, :, :, num_anchors:])) / K.sum(epsilon + y_true[:, :, :, :num_anchors])\n\n\treturn rpn_loss_cls_fixed_num\n\n\ndef class_loss_regr(num_classes):\n\tdef class_loss_regr_fixed_num(y_true, y_pred):\n\t\tx = y_true[:, :, 4*num_classes:] - y_pred\n\t\tx_abs = K.abs(x)\n\t\tx_bool = K.cast(K.less_equal(x_abs, 1.0), 'float32')\n\t\treturn lambda_cls_regr * K.sum(y_true[:, :, :4*num_classes] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :4*num_classes])\n\treturn class_loss_regr_fixed_num\n\n\ndef class_loss_cls(y_true, y_pred):\n\treturn lambda_cls_class * K.mean(categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn_weight_path = './rpn_vgg.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-3\n# if rpn_weight_path is not None:\noptimizer = SGD(lr=lr/100, decay=0.0005, momentum=0.9)\noptimizer_classifier = SGD(lr=lr/5, decay=0.0005, momentum=0.9)\n# else:\n#     optimizer = SGD(lr=lr/10, decay=0.0005, momentum=0.9)\n#     optimizer_classifier = SGD(lr=lr/10, decay=0.0005, momentum=0.9)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C.base_net_weights = '../input/faster-rcnn/full_model_frcnn.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('weights loaded from full faster rcnn')\nmodel_rpn.load_weights(C.base_net_weights, by_name=True)\nmodel_classifier.load_weights(C.base_net_weights, by_name=True)\n# print(\"loading RPN weights from \", rpn_weight_path)\n# model_rpn.load_weights(rpn_weight_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compile the model AFTER loading weights!\nmodel_rpn.compile(optimizer=optimizer, loss=[rpn_loss_cls(num_anchors), rpn_loss_regr(num_anchors)])\nmodel_classifier.compile(optimizer=optimizer_classifier, loss=[class_loss_cls, class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})\nmodel_all.compile(optimizer='sgd', loss='mae')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimizer = Adam(lr=1e-5)\n# optimizer_classifier = Adam(lr=1e-5) # learning rate should be less\n\n# model_rpn.compile(optimizer=optimizer, loss=[rpn_loss_cls(num_anchors), rpn_loss_regr(num_anchors)])\n# model_classifier.compile(optimizer=optimizer_classifier, loss=[class_loss_cls, class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})\n# model_all.compile(optimizer='sgd', loss='mae')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function):\n    downscale = float(C.rpn_stride) \n    anchor_sizes = C.anchor_box_scales   # 128, 256, 512\n    anchor_ratios = C.anchor_box_ratios  # 1:1, 1:2*sqrt(2), 2*sqrt(2):1\n    num_anchors = len(anchor_sizes) * len(anchor_ratios)\n    (output_width, output_height) = img_length_calc_function(resized_width, resized_height)\n    n_anchratios = len(anchor_ratios)\n    y_rpn_overlap = np.zeros((output_height, output_width, num_anchors))\n    y_is_box_valid = np.zeros((output_height, output_width, num_anchors))\n    y_rpn_regr = np.zeros((output_height, output_width, num_anchors * 4))\n    num_bboxes = len(img_data['bboxes'])\n    num_anchors_for_bbox = np.zeros(num_bboxes).astype(int)\n    best_anchor_for_bbox = -1*np.ones((num_bboxes, 4)).astype(int)\n    best_iou_for_bbox = np.zeros(num_bboxes).astype(np.float32)\n    best_x_for_bbox = np.zeros((num_bboxes, 4)).astype(int)\n    best_dx_for_bbox = np.zeros((num_bboxes, 4)).astype(np.float32)\n    gta = np.zeros((num_bboxes, 4))\n    for bbox_num, bbox in enumerate(img_data['bboxes']):\n        # get the GT box coordinates, and resize to account for image resizing\n        gta[bbox_num, 0] = bbox['x1'] * (resized_width / float(width))\n        gta[bbox_num, 1] = bbox['x2'] * (resized_width / float(width))\n        gta[bbox_num, 2] = bbox['y1'] * (resized_height / float(height))\n        gta[bbox_num, 3] = bbox['y2'] * (resized_height / float(height))\n        \n#     print (anchor_ratios)\n        \n    for anchor_size_idx in range(len(anchor_sizes)):\n        for anchor_ratio_idx in range(n_anchratios):\n            anchor_x = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][0]\n            anchor_y = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][1]\t\n#             print (anchor_x, anchor_y) gives 9 anchor w and h\n            \n            for ix in range(output_width):\t\t\t\t\t\n                # x-coordinates of the current anchor box\t\n                x1_anc = downscale * (ix + 0.5) - anchor_x / 2\n                x2_anc = downscale * (ix + 0.5) + anchor_x / 2\n#                 print (x1_anc, x2_anc)\n                \n                if x1_anc < 0 or x2_anc > resized_width:# ignore boxes that go across image boundaries\t\n                    continue\n                for jy in range(output_height):\n                    # y-coordinates of the current anchor box\n                    y1_anc = downscale * (jy + 0.5) - anchor_y / 2\n                    y2_anc = downscale * (jy + 0.5) + anchor_y / 2\n\n                    # ignore boxes that go across image boundaries\n                    if y1_anc < 0 or y2_anc > resized_height:\n                        continue\n                        \n                    bbox_type = 'neg'\n                    best_iou_for_loc = 0.0\n                    for bbox_num in range(num_bboxes):\n                        curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1_anc, y1_anc, x2_anc, y2_anc])\n                        if curr_iou > best_iou_for_bbox[bbox_num] or curr_iou > C.rpn_max_overlap:\n                            cx = (gta[bbox_num, 0] + gta[bbox_num, 1]) / 2.0\n                            cy = (gta[bbox_num, 2] + gta[bbox_num, 3]) / 2.0\n                            cxa = (x1_anc + x2_anc)/2.0\n                            cya = (y1_anc + y2_anc)/2.0\n                            tx = (cx - cxa) / (x2_anc - x1_anc)\n                            ty = (cy - cya) / (y2_anc - y1_anc)\n                            tw = np.log((gta[bbox_num, 1] - gta[bbox_num, 0]) / (x2_anc - x1_anc))\n                            th = np.log((gta[bbox_num, 3] - gta[bbox_num, 2]) / (y2_anc - y1_anc))\n#                             print (\"gt co-ordinates:\"+\"center :\"+str(cx)+\" ,\"+str(cy) +\"x1,x2,y1,y2 :\"+  str(gta[bbox_num,:]))\n#                             print (\"anchor co-ordinates:\"+\"center :\"+str(cxa)+\" ,\"+str(cya) +\"x1,x2,y1,y2 :\"+ str(x1_anc)+\" ,\"+str(x2_anc)+\" ,\"+str(y1_anc)+\" ,\"+str(y2_anc))\n#                             print (tx,ty,tw,th)\n                        if img_data['bboxes'][bbox_num]['class'] != 'bg':\n                            if curr_iou > best_iou_for_bbox[bbox_num]:\n                                best_anchor_for_bbox[bbox_num] = [jy, ix, anchor_ratio_idx, anchor_size_idx]\n                                best_iou_for_bbox[bbox_num] = curr_iou\n                                best_x_for_bbox[bbox_num,:] = [x1_anc, x2_anc, y1_anc, y2_anc]\n                                best_dx_for_bbox[bbox_num,:] = [tx, ty, tw, th]\n                                \n                            if curr_iou > C.rpn_max_overlap:\n                                bbox_type = 'pos'\n                                num_anchors_for_bbox[bbox_num] += 1\n                                if curr_iou > best_iou_for_loc:\n                                    best_iou_for_loc = curr_iou\n                                    best_regr = (tx, ty, tw, th)\n                            if C.rpn_min_overlap < curr_iou < C.rpn_max_overlap:\n                                # gray zone between neg and pos\n                                if bbox_type != 'pos':\n                                    bbox_type = 'neutral'\n                    if bbox_type == 'neg':\n                        y_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n                        y_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n                    elif bbox_type == 'neutral':\n                        y_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n                        y_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n                    elif bbox_type == 'pos':\n                        y_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n                        y_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n                        start = 4 * (anchor_ratio_idx + n_anchratios * anchor_size_idx)\n                        y_rpn_regr[jy, ix, start:start+4] = best_regr\n\n\n    for idx in range(num_anchors_for_bbox.shape[0]):\n        if num_anchors_for_bbox[idx] == 0:\n            # no box with an IOU greater than zero ...\n            if best_anchor_for_bbox[idx, 0] == -1:\n                continue\n            y_is_box_valid[best_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3]] = 1\n            y_rpn_overlap[best_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3]] = 1\n            start = 4 * (best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3])\n            y_rpn_regr[best_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], start:start+4] = best_dx_for_bbox[idx, :]\n\n    y_rpn_overlap = np.transpose(y_rpn_overlap, (2, 0, 1))\n    y_rpn_overlap = np.expand_dims(y_rpn_overlap, axis=0)\n\n    y_is_box_valid = np.transpose(y_is_box_valid, (2, 0, 1))\n    y_is_box_valid = np.expand_dims(y_is_box_valid, axis=0)\n\n    y_rpn_regr = np.transpose(y_rpn_regr, (2, 0, 1))\n    y_rpn_regr = np.expand_dims(y_rpn_regr, axis=0)\n\n    pos_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1))\n    neg_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 0, y_is_box_valid[0, :, :, :] == 1))\n    \n    num_pos = len(pos_locs[0])\n    \n    num_regions = 256\n\n    if len(pos_locs[0]) > num_regions/2:\n        val_locs = random.sample(range(len(pos_locs[0])), len(pos_locs[0]) - num_regions/2)\n        y_is_box_valid[0, pos_locs[0][val_locs], pos_locs[1][val_locs], pos_locs[2][val_locs]] = 0\n        num_pos = num_regions/2\n\n    if len(neg_locs[0]) + num_pos > num_regions:\n        val_locs = random.sample(range(len(neg_locs[0])), len(neg_locs[0]) - num_pos)\n        y_is_box_valid[0, neg_locs[0][val_locs], neg_locs[1][val_locs], neg_locs[2][val_locs]] = 0\n\n    y_rpn_cls = np.concatenate([y_is_box_valid, y_rpn_overlap], axis=1)\n    y_rpn_regr = np.concatenate([np.repeat(y_rpn_overlap, 4, axis=1), y_rpn_regr], axis=1)\n#     print (y_rpn_cls.shape)\n#     print (y_rpn_regr.shape)\n    return np.copy(y_rpn_cls), np.copy(y_rpn_regr), num_pos\n\n#     x = np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1)\n#     count = 0\n#     for i in range(0,9):\n#         for j in range(0,output_width):\n#             for k in range(0,output_width):\n#                 if x[i,j,k] == True:\n#                     count = count + 1\n#     print (count)\n    \n    \n#     print (x.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def union(au, bu, area_intersection):\n    area_a = (au[2] - au[0]) * (au[3] - au[1])\n    area_b = (bu[2] - bu[0]) * (bu[3] - bu[1])\n    area_union = area_a + area_b - area_intersection\n    return area_union\n\n\ndef intersection(ai, bi):\n    x = max(ai[0], bi[0])\n    y = max(ai[1], bi[1])\n    w = min(ai[2], bi[2]) - x\n    h = min(ai[3], bi[3]) - y\n    if w < 0 or h < 0:\n        return 0\n    return w*h\n\n\ndef iou(a, b):\n\t# a and b should be (x1,y1,x2,y2)\n\n    if a[0] >= a[2] or a[1] >= a[3] or b[0] >= b[2] or b[1] >= b[3]:\n        return 0.0\n\n    area_i = intersection(a, b)\n    area_u = union(a, b, area_i)\n\n    return float(area_i) / float(area_u + 1e-6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_new_img_size(width, height, img_min_side=300):\n\tif width <= height:\n\t\tf = float(img_min_side) / width\n\t\tresized_height = int(f * height)\n\t\tresized_width = img_min_side\n\telse:\n\t\tf = float(img_min_side) / height\n\t\tresized_width = int(f * width)\n\t\tresized_height = img_min_side\n\n\treturn resized_width, resized_height\n\ndef augment(img_data, config, augment=True):\n\tassert 'filepath' in img_data\n\tassert 'bboxes' in img_data\n\tassert 'width' in img_data\n\tassert 'height' in img_data\n\n\timg_data_aug = copy.deepcopy(img_data)\n\n\timg = cv2.imread(img_data_aug['filepath'])\n\n\tif augment:\n\t\trows, cols = img.shape[:2]\n\n\t\tif config.use_horizontal_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 1)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\tbbox['x1'] = cols - x2\n\n\t\tif config.use_vertical_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 0)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\tbbox['y1'] = rows - y2\n\n\t\tif config.rot_90:\n\t\t\tangle = np.random.choice([0,90,180,270],1)[0]\n\t\t\tif angle == 270:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 0)\n\t\t\telif angle == 180:\n\t\t\t\timg = cv2.flip(img, -1)\n\t\t\telif angle == 90:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 1)\n\t\t\telif angle == 0:\n\t\t\t\tpass\n\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tif angle == 270:\n\t\t\t\t\tbbox['x1'] = y1\n\t\t\t\t\tbbox['x2'] = y2\n\t\t\t\t\tbbox['y1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = cols - x1\n\t\t\t\telif angle == 180:\n\t\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\t\tbbox['x1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = rows - y2\n\t\t\t\telif angle == 90:\n\t\t\t\t\tbbox['x1'] = rows - y2\n\t\t\t\t\tbbox['x2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = x1\n\t\t\t\t\tbbox['y2'] = x2        \n\t\t\t\telif angle == 0:\n\t\t\t\t\tpass\n\n\timg_data_aug['width'] = img.shape[1]\n\timg_data_aug['height'] = img.shape[0]\n\treturn img_data_aug, img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_anchor_gt(all_img_data, C, img_length_calc_function, mode='train'):\n\t\"\"\" Yield the ground-truth anchors as Y (labels)\n\t\t\n\tArgs:\n\t\tall_img_data: list(filepath, width, height, list(bboxes))\n\t\tC: config\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\t\tmode: 'train' or 'test'; 'train' mode need augmentation\n\n\tReturns:\n\t\tx_img: image data after resized and scaling (smallest size = 300px)\n\t\tY: [y_rpn_cls, y_rpn_regr]\n\t\timg_data_aug: augmented image data (original image with augmentation)\n\t\tdebug_img: show image for debug\n\t\tnum_pos: show number of positive anchors for debug\n\t\"\"\"\n\twhile True:\n\n\t\tfor img_data in all_img_data:\n\t\t\ttry:\n\n\t\t\t\tx_img = cv2.imread(img_data['filepath'])\n\n# \t\t\t\tif mode == 'train':\n# \t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=True)\n# \t\t\t\telse:\n# \t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=False)\n\n\t\t\t\t(width, height) = (img_data['width'], img_data['height'])\n\t\t\t\t(rows, cols, _) = x_img.shape\n\n\t\t\t\tassert cols == width\n\t\t\t\tassert rows == height\n\n\t\t\t\t# get image dimensions for resizing\n\t\t\t\t(resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n\t\t\t\t# resize the image so that smalles side is length = 300px\n\t\t\t\tx_img = cv2.resize(x_img, (resized_width, resized_height), interpolation=cv2.INTER_CUBIC)\n\t\t\t\tdebug_img = x_img.copy()\n#                 print (debug_img)\n\n\t\t\t\ttry:\n\t\t\t\t\ty_rpn_cls, y_rpn_regr, num_pos = calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function)\n\t\t\t\texcept:\n\t\t\t\t\tcontinue\n\n\t\t\t\t# Zero-center by mean pixel, and preprocess image\n\n\t\t\t\tx_img = x_img[:,:, (2, 1, 0)]  # BGR -> RGB\n\t\t\t\tx_img = x_img.astype(np.float32)\n\t\t\t\tx_img[:, :, 0] -= C.img_channel_mean[0]\n\t\t\t\tx_img[:, :, 1] -= C.img_channel_mean[1]\n\t\t\t\tx_img[:, :, 2] -= C.img_channel_mean[2]\n\t\t\t\tx_img /= C.img_scaling_factor\n\n\t\t\t\tx_img = np.transpose(x_img, (2, 0, 1))\n\t\t\t\tx_img = np.expand_dims(x_img, axis=0)\n\n\t\t\t\ty_rpn_regr[:, y_rpn_regr.shape[1]//2:, :, :] *= C.std_scaling\n\n\t\t\t\tx_img = np.transpose(x_img, (0, 2, 3, 1))\n\t\t\t\ty_rpn_cls = np.transpose(y_rpn_cls, (0, 2, 3, 1))\n\t\t\t\ty_rpn_regr = np.transpose(y_rpn_regr, (0, 2, 3, 1))\n\n\t\t\t\tyield np.copy(x_img), [np.copy(y_rpn_cls), np.copy(y_rpn_regr)], img_data, debug_img, num_pos\n\n\t\t\texcept Exception as e:\n\t\t\t\tprint(e)\n\t\t\t\tcontinue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_output_length(width, height):\n    def get_output_length(input_length):\n        return input_length//16\n\n    return get_output_length(width), get_output_length(height) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SampleSelector:\n    def __init__(self, class_count):\n        # ignore classes that have zero samples\n        self.classes = [b for b in class_count.keys() if class_count[b] > 0]\n        self.class_cycle = itertools.cycle(self.classes)\n        self.curr_class = next(self.class_cycle)\n\n    def skip_sample_for_balanced_class(self, img_data):\n\n        class_in_img = False\n\n        for bbox in img_data['bboxes']:\n\n            cls_name = bbox['class']\n#             print (cls_name)\n#             print ('end')\n#             print (self.curr_class)\n\n            if cls_name == self.curr_class:\n                class_in_img = True\n                self.curr_class = next(self.class_cycle)\n                break\n\n        if class_in_img:\n            return False\n        else:\n            return True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_new_img_size(width, height, img_min_side=600):\n\tif width <= height:\n\t\tf = float(img_min_side) / width\n\t\tresized_height = int(f * height)\n\t\tresized_width = img_min_side\n\telse:\n\t\tf = float(img_min_side) / height\n\t\tresized_width = int(f * width)\n\t\tresized_height = img_min_side\n\n\treturn resized_width, resized_height\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_gen_train = get_anchor_gt(train_imgs, C, get_img_output_length, mode='train')\n# X, Y, image_data, debug_img, debug_num_pos = next(data_gen_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# C.model_path = ''\n# C.base_net_weights = './vgg16_weights_tf_dim_ordering_tf_kernels.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if not os.path.isfile(C.model_path):\n#     #If this is the begin of the training, load the pre-traind base network such as vgg-16\n#     try:\n#         print('This is the first time of your training')\n#         print('loading weights from {}'.format(C.base_net_weights))\n#         model_rpn.load_weights(C.base_net_weights, by_name=True)\n#         model_classifier.load_weights(C.base_net_weights, by_name=True)\n#     except:\n#         print('Could not load pretrained model weights. Weights can be found in the keras application folder \\\n#             https://github.com/fchollet/keras/tree/master/keras/applications')\n    \n#     # Create the record.csv file to record losses, acc and mAP\n#     record_df = pd.DataFrame(columns=['mean_overlapping_bboxes', 'class_acc', 'loss_rpn_cls', 'loss_rpn_regr', 'loss_class_cls', 'loss_class_regr', 'curr_loss', 'elapsed_time', 'mAP'])\n# else:\n#     # If this is a continued training, load the trained model from before\n#     print('Continue training based on previous trained model')\n#     print('Loading weights from {}'.format(C.model_path))\n# #     model_rpn.load_weights(C.model_path, by_name=True)\n# #     model_classifier.load_weights(C.model_path, by_name=True)\n\n#     model_rpn.load_weights(C.model_path)\n#     model_classifier.load_weights(C.model_path)    \n\n    \n# #     # Load the records\n# #     record_df = pd.read_csv(record_path)\n\n# #     r_mean_overlapping_bboxes = record_df['mean_overlapping_bboxes']\n# #     r_class_acc = record_df['class_acc']\n# #     r_loss_rpn_cls = record_df['loss_rpn_cls']\n# #     r_loss_rpn_regr = record_df['loss_rpn_regr']\n# #     r_loss_class_cls = record_df['loss_class_cls']\n# #     r_loss_class_regr = record_df['loss_class_regr']\n# #     r_curr_loss = record_df['curr_loss']\n# #     r_elapsed_time = record_df['elapsed_time']\n# #     r_mAP = record_df['mAP']\n\n# #     print('Already train %dK batches'% (len(record_df)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):\n\t\"\"\"Convert rpn layer to roi bboxes\n\n\tArgs: (num_anchors = 9)\n\t\trpn_layer: output layer for rpn classification \n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 18) if resized image is 400 width and 300\n\t\tregr_layer: output layer for rpn regression\n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 72) if resized image is 400 width and 300\n\t\tC: config\n\t\tuse_regr: Wether to use bboxes regression in rpn\n\t\tmax_boxes: max bboxes number for non-max-suppression (NMS)\n\t\toverlap_thresh: If iou in NMS is larger than this threshold, drop the box\n\n\tReturns:\n\t\tresult: boxes from non-max-suppression (shape=(300, 4))\n\t\t\tboxes: coordinates for bboxes (on the feature map)\n\t\"\"\"\n\tregr_layer = regr_layer / C.std_scaling\n\n\tanchor_sizes = C.anchor_box_scales   # (3 in here)\n\tanchor_ratios = C.anchor_box_ratios  # (3 in here)\n\n\tassert rpn_layer.shape[0] == 1\n\n\t(rows, cols) = rpn_layer.shape[1:3]\n\n\tcurr_layer = 0\n\n\t# A.shape = (4, feature_map.height, feature_map.width, num_anchors) \n\t# Might be (4, 18, 25, 18) if resized image is 400 width and 300\n\t# A is the coordinates for 9 anchors for every point in the feature map \n\t# => all 18x25x9=4050 anchors cooridnates\n\tA = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))\n\n\tfor anchor_size in anchor_sizes:\n\t\tfor anchor_ratio in anchor_ratios:\n\t\t\t# anchor_x = (128 * 1) / 16 = 8  => width of current anchor\n\t\t\t# anchor_y = (128 * 2) / 16 = 16 => height of current anchor\n\t\t\tanchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride\n\t\t\tanchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride\n\t\t\t\n\t\t\t# curr_layer: 0~8 (9 anchors)\n\t\t\t# the Kth anchor of all position in the feature map (9th in total)\n\t\t\tregr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4] # shape => (18, 25, 4)\n\t\t\tregr = np.transpose(regr, (2, 0, 1)) # shape => (4, 18, 25)\n\n\t\t\t# Create 18x25 mesh grid\n\t\t\t# For every point in x, there are all the y points and vice versa\n\t\t\t# X.shape = (18, 25)\n\t\t\t# Y.shape = (18, 25)\n\t\t\tX, Y = np.meshgrid(np.arange(cols),np. arange(rows))\n\n\t\t\tprint (X, Y)\n\t\t\tA[0, :, :, curr_layer] = X - anchor_x/2 # Top left x coordinate\n\t\t\tA[1, :, :, curr_layer] = Y - anchor_y/2 # Top left y coordinate\n\t\t\tA[2, :, :, curr_layer] = anchor_x       # width of current anchor\n\t\t\tA[3, :, :, curr_layer] = anchor_y       # height of current anchor\n\n\t\t\t# Apply regression to x, y, w and h if there is rpn regression layer\n\t\t\tif use_regr:\n\t\t\t\tA[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)\n\n\t\t\t# Avoid width and height exceeding 1\n\t\t\tA[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])\n\n\t\t\t# Convert (x, y , w, h) to (x1, y1, x2, y2)\n\t\t\t# x1, y1 is top left coordinate\n\t\t\t# x2, y2 is bottom right coordinate\n\t\t\tA[2, :, :, curr_layer] += A[0, :, :, curr_layer]\n\t\t\tA[3, :, :, curr_layer] += A[1, :, :, curr_layer]\n\n\t\t\t# Avoid bboxes drawn outside the feature map\n\t\t\tA[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])\n\t\t\tA[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])\n\t\t\tA[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])\n\n\t\t\tcurr_layer += 1\n\n\tall_boxes = np.reshape(A.transpose((0, 3, 1, 2)), (4, -1)).transpose((1, 0))  # shape=(4050, 4)\n\tall_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))                   # shape=(4050,)\n\n\tx1 = all_boxes[:, 0]\n\ty1 = all_boxes[:, 1]\n\tx2 = all_boxes[:, 2]\n\ty2 = all_boxes[:, 3]\n\n\t# Find out the bboxes which is illegal and delete them from bboxes list\n\tidxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))\n\n\tall_boxes = np.delete(all_boxes, idxs, 0)\n\tall_probs = np.delete(all_probs, idxs, 0)\n\n\t# Apply non_max_suppression\n\t# Only extract the bboxes. Don't need rpn probs in the later process\n\tresult = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]\n\n\treturn result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_iou(R, img_data, C, class_mapping):\n    \"\"\"Converts from (x1,y1,x2,y2) to (x,y,w,h) format\n\n    Args:\n        R: bboxes, probs\n    \"\"\"\n    bboxes = img_data['bboxes']\n    (width, height) = (img_data['width'], img_data['height'])\n    # get image dimensions for resizing\n    (resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n    gta = np.zeros((len(bboxes), 4))\n\n    for bbox_num, bbox in enumerate(bboxes):\n        # get the GT box coordinates, and resize to account for image resizing\n        # gta[bbox_num, 0] = (40 * (600 / 800)) / 16 = int(round(1.875)) = 2 (x in feature map)\n        gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))\n        gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))\n        \n    x_roi = []\n    y_class_num = []\n    y_class_regr_coords = []\n    y_class_regr_label = []\n    IoUs = [] # for debugging only\n    for ix in range(R.shape[0]):\n        (x1, y1, x2, y2) = R[ix, :]\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        x2 = int(round(x2))\n        y2 = int(round(y2))\n\n        best_iou = 0.0\n        best_bbox = -1\n        for bbox_num in range(len(bboxes)):\n            curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])\n#             print (curr_iou)\n            if curr_iou > best_iou:\n                best_iou = curr_iou\n                best_bbox = bbox_num\n#         print (best_iou)\n        if best_iou < C.classifier_min_overlap:\n#             print ('nope')\n            continue\n        else:\n            w = x2 - x1\n            h = y2 - y1\n            x_roi.append([x1, y1, w, h])\n            IoUs.append(best_iou)\n            \n            if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:\n                # hard negative example\n                cls_name = 'bg'\n            else:\n                w = x2 - x1\n                h = y2 - y1\n                x_roi.append([x1, y1, w, h])\n                IoUs.append(best_iou)\n\n                if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:\n                    # hard negative example\n                    cls_name = 'bg'\n                elif C.classifier_max_overlap <= best_iou:\n                    cls_name = bboxes[best_bbox]['class']\n                    cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0\n                    cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0\n\n                    cx = x1 + w / 2.0\n                    cy = y1 + h / 2.0\n\n                    tx = (cxg - cx) / float(w)\n                    ty = (cyg - cy) / float(h)\n                    tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))\n                    th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))\n                else:\n                    print('roi = {}'.format(best_iou))\n                    raise RuntimeError\n        class_num = class_mapping[cls_name]\n        class_label = len(class_mapping) * [0]\n#         print (class_num)\n        class_label[class_num] = 1\n        y_class_num.append(copy.deepcopy(class_label))\n        coords = [0] * 4 * (len(class_mapping) - 1)\n        labels = [0] * 4 * (len(class_mapping) - 1)\n        if cls_name != 'bg':\n            label_pos = 4 * class_num\n            sx, sy, sw, sh = C.classifier_regr_std\n            coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]\n            labels[label_pos:4+label_pos] = [1, 1, 1, 1]\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n        else:\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n\n    if len(x_roi) == 0:\n        return None, None, None, None\n    X = np.array(x_roi)\n    # one hot code for bboxes from above => x_roi (X)\n    Y1 = np.array(y_class_num)\n    # corresponding labels and corresponding gt bboxes\n    Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)\n\n    return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0), IoUs\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_regr_np(X, T):\n    try:\n        x = X[0, :, :]\n        y = X[1, :, :]\n        w = X[2, :, :]\n        h = X[3, :, :]\n\n        tx = T[0, :, :]\n        ty = T[1, :, :]\n        tw = T[2, :, :]\n        th = T[3, :, :]\n\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n\n        w1 = np.exp(tw.astype(np.float64)) * w\n        h1 = np.exp(th.astype(np.float64)) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n\n        x1 = np.round(x1)\n        y1 = np.round(y1)\n        w1 = np.round(w1)\n        h1 = np.round(h1)\n        return np.stack([x1, y1, w1, h1])\n    except Exception as e:\n        print(e)\n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):\n    # code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/\n    # if there are no boxes, return an empty list\n\n    # Process explanation:\n    #   Step 1: Sort the probs list\n    #   Step 2: Find the larget prob 'Last' in the list and save it to the pick list\n    #   Step 3: Calculate the IoU with 'Last' box and other boxes in the list. If the IoU is larger than overlap_threshold, delete the box from list\n    #   Step 4: Repeat step 2 and step 3 until there is no item in the probs list \n    if len(boxes) == 0:\n        return []\n\n    # grab the coordinates of the bounding boxes\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    np.testing.assert_array_less(x1, x2)\n    np.testing.assert_array_less(y1, y2)\n\n    # if the bounding boxes integers, convert them to floats --\n    # this is important since we'll be doing a bunch of divisions\n    if boxes.dtype.kind == \"i\":\n        boxes = boxes.astype(\"float\")\n\n    # initialize the list of picked indexes\t\n    pick = []\n\n    # calculate the areas\n    area = (x2 - x1) * (y2 - y1)\n\n    # sort the bounding boxes \n    idxs = np.argsort(probs)\n\n    # keep looping while some indexes still remain in the indexes\n    # list\n    while len(idxs) > 0:\n        # grab the last index in the indexes list and add the\n        # index value to the list of picked indexes\n        last = len(idxs) - 1\n        i = idxs[last]\n        pick.append(i)\n\n        # find the intersection\n\n        xx1_int = np.maximum(x1[i], x1[idxs[:last]])\n        yy1_int = np.maximum(y1[i], y1[idxs[:last]])\n        xx2_int = np.minimum(x2[i], x2[idxs[:last]])\n        yy2_int = np.minimum(y2[i], y2[idxs[:last]])\n\n        ww_int = np.maximum(0, xx2_int - xx1_int)\n        hh_int = np.maximum(0, yy2_int - yy1_int)\n\n        area_int = ww_int * hh_int\n\n        # find the union\n        area_union = area[i] + area[idxs[:last]] - area_int\n\n        # compute the ratio of overlap\n        overlap = area_int/(area_union + 1e-6)\n\n        # delete all indexes from the index list that have\n        idxs = np.delete(idxs, np.concatenate(([last],\n            np.where(overlap > overlap_thresh)[0])))\n\n        if len(pick) >= max_boxes:\n            break\n\n    # return only the bounding boxes that were picked using the integer data type\n    boxes = boxes[pick].astype(\"int\")\n    probs = probs[pick]\n    return boxes, probs\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):\n#     print (regr_layer[0,0,0,:])\n    regr_layer = regr_layer / C.std_scaling\n\n    anchor_sizes = C.anchor_box_scales   # (3 in here)\n    anchor_ratios = C.anchor_box_ratios  # (3 in here)\n\n    assert rpn_layer.shape[0] == 1\n    (rows, cols) = rpn_layer.shape[1:3]\n    curr_layer = 0\n    A = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))\n#     print (A.shape)\n    for anchor_size in anchor_sizes:\n        for anchor_ratio in anchor_ratios:\n#             print (\"current_layer : \"+str(curr_layer))\n            # anchor_x = (128 * 1) / 16 = 8  => width of current anchor\n            # anchor_y = (128 * 2) / 16 = 16 => height of current anchor\n            anchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride\n            anchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride\n            \n            # curr_layer: 0~8 (9 anchors)\n            # the Kth anchor of all position in the feature map (9th in total)\n            regr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4] # shape => (18, 25, 4)\n\n            regr = np.transpose(regr, (2, 0, 1)) # shape => (4, 18, 25)\n            X, Y = np.meshgrid(np.arange(cols),np.arange(rows))\n            A[0, :, :, curr_layer] = X - anchor_x/2 # Top left x coordinate\n            A[1, :, :, curr_layer] = Y - anchor_y/2 # Top left y coordinate\n            A[2, :, :, curr_layer] = anchor_x       # width of current anchor\n            A[3, :, :, curr_layer] = anchor_y\n            if use_regr:\n                A[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)\n#             print (A[2,:,:,curr_layer])\n            A[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])\n            A[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])\n            A[2, :, :, curr_layer] += A[0, :, :, curr_layer]\n            A[3, :, :, curr_layer] += A[1, :, :, curr_layer]\n            A[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])\n            A[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])\n            A[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])\n            A[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])\n            curr_layer = curr_layer + 1\n#             print (A[:,0,0,curr_layer])\n    all_boxes = np.reshape(A.transpose((0, 3, 1, 2)), (4, -1)).transpose((1, 0))  # shape=(37*37*9, 4)\n    all_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1)) \n#     print (all_boxes.shape)\n#     print (all_probs.shape)\n    x1 = all_boxes[:, 0]\n    y1 = all_boxes[:, 1]\n    x2 = all_boxes[:, 2]\n    y2 = all_boxes[:, 3]\n    idxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))\n    all_boxes = np.delete(all_boxes, idxs, 0)\n    all_probs = np.delete(all_probs, idxs, 0)\n#     print (all_boxes)\n#     print (all_probs)\n    result = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]\n    return result\n            \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.listdir('../input/voctrain/VOCdevkit/VOC2007/JPEGImages/')[116]\n# img = cv2.imread('../input/voctrain/VOCdevkit/VOC2007/JPEGImages/009959.jpg')\n# img1 = np.expand_dims(img, axis = 0)\n# # X, ratio = format_img(img, C)\n# # X = np.transpose(X, (0, 2, 3, 1))\n# [Y1, Y2] = model_rpn.predict(img1)\n\n# R = rpn_to_roi(Y1, Y2, C, 'tf', overlap_thresh=0.7)\n\n# R[:, 2] -= R[:, 0]\n# R[:, 3] -= R[:, 1]\n# # R = np.expand_dims(R[:,:], axis=0)\n# print (R.shape)\n# import cv2\n# import matplotlib.pyplot as plt\n# %matplotlib inline\n\n# for i in range(0,300):\n\n#     # Draw the rectangles\n#     cv2.rectangle(img1[0], (int(R[i,0]*16),int(R[i,1]*16)), (int(R[i,2]*16), int(R[i,2]*16)), (0, 255, 0), 3) \n\n# plt.imshow(img1[0])\n# # cv2.waitKey()\n\n# ROIs = np.expand_dims(R, axis=0)\n# [P_cls, P_regr] = model_classifier.predict([img1, ROIs])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# total_epochs = len(record_df)\n# r_epochs = len(record_df)\nr_epochs = 0\ntotal_epochs = 0\nepoch_length = 1000\nnum_epochs = 30\niter_num = 0\n\ntotal_epochs += num_epochs\n\n# total_epochs = 1\nlosses = np.zeros((epoch_length, 5))\nrpn_accuracy_rpn_monitor = []\nrpn_accuracy_for_epoch = []\n\n# if len(record_df)==0:\nbest_loss = np.Inf\n# else:\n#     best_loss = np.min(r_curr_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C.model_path = '/kaggle/working/full_model_frcnn.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nfor epoch_num in range(num_epochs):\n\n    progbar = generic_utils.Progbar(epoch_length)\n    print('Epoch {}/{}'.format(r_epochs + 1, total_epochs))\n    \n    r_epochs += 1\n\n    while True:\n        try:\n\n            if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:\n                mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor))/len(rpn_accuracy_rpn_monitor)\n                rpn_accuracy_rpn_monitor = []\n#                 print('Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'.format(mean_overlapping_bboxes, epoch_length))\n                if mean_overlapping_bboxes == 0:\n                    print('RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.')\n\n            # Generate X (x_img) and label Y ([y_rpn_cls, y_rpn_regr])\n            X, Y, img_data, debug_img, debug_num_pos = next(data_gen_train)\n\n            # Train rpn model and get loss value [_, loss_rpn_cls, loss_rpn_regr]\n            \n#             print (model_rpn.evaluate(X,y))\n            \n            loss_rpn = model_rpn.train_on_batch(X, Y)\n\n            # Get predicted rpn from rpn model [rpn_cls, rpn_regr]\n            P_rpn = model_rpn.predict_on_batch(X)\n\n            # R: bboxes (shape=(300,4))\n            # Convert rpn layer to roi bboxes\n            R = rpn_to_roi(P_rpn[0], P_rpn[1], C, 'tf', use_regr=True, overlap_thresh=0.7, max_boxes=300)\n            \n            # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format\n            # X2: bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n            # Y1: one hot code for bboxes from above => x_roi (X)\n            # Y2: corresponding labels and corresponding gt bboxes\n            X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping)\n\n            # If X2 is None means there are no matching bboxes\n            if X2 is None:\n                rpn_accuracy_rpn_monitor.append(0)\n                rpn_accuracy_for_epoch.append(0)\n                continue\n            \n            # Find out the positive anchors and negative anchors\n            neg_samples = np.where(Y1[0, :, -1] == 1)\n            pos_samples = np.where(Y1[0, :, -1] == 0)\n#             if (epoch_length == 100):\n#                 print (\"Length of positive samples :\")\n#                 print (len(pos_samples))\n\n            if len(neg_samples) > 0:\n                neg_samples = neg_samples[0]\n            else:\n                neg_samples = []\n\n            if len(pos_samples) > 0:\n                pos_samples = pos_samples[0]\n            else:\n                pos_samples = []\n\n            rpn_accuracy_rpn_monitor.append(len(pos_samples))\n            rpn_accuracy_for_epoch.append((len(pos_samples)))\n\n            if C.num_rois > 1:\n                # If number of positive anchors is larger than 4//2 = 2, randomly choose 2 pos samples\n                if len(pos_samples) < C.num_rois//2:\n                    selected_pos_samples = pos_samples.tolist()\n                else:\n                    selected_pos_samples = np.random.choice(pos_samples, C.num_rois//2, replace=False).tolist()\n                \n                # Randomly choose (num_rois - num_pos) neg samples\n                try:\n                    selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist()\n                except:\n                    selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist()\n                \n                # Save all the pos and neg samples in sel_samples\n                sel_samples = selected_pos_samples + selected_neg_samples\n            else:\n                # in the extreme case where num_rois = 1, we pick a random pos or neg sample\n                selected_pos_samples = pos_samples.tolist()\n                selected_neg_samples = neg_samples.tolist()\n                if np.random.randint(0, 2):\n                    sel_samples = random.choice(neg_samples)\n                else:\n                    sel_samples = random.choice(pos_samples)\n\n            # training_data: [X, X2[:, sel_samples, :]]\n            # labels: [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]\n            #  X                     => img_data resized image\n            #  X2[:, sel_samples, :] => num_rois (4 in here) bboxes which contains selected neg and pos\n            #  Y1[:, sel_samples, :] => one hot encode for num_rois bboxes which contains selected neg and pos\n            #  Y2[:, sel_samples, :] => labels and gt bboxes for num_rois bboxes which contains selected neg and pos\n            loss_class = model_classifier.train_on_batch([X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]])\n\n            losses[iter_num, 0] = loss_rpn[1]\n            losses[iter_num, 1] = loss_rpn[2]\n\n            losses[iter_num, 2] = loss_class[1]\n            losses[iter_num, 3] = loss_class[2]\n            losses[iter_num, 4] = loss_class[3]\n\n            iter_num += 1\n\n            progbar.update(iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])),\n                                      ('final_cls', np.mean(losses[:iter_num, 2])), ('final_regr', np.mean(losses[:iter_num, 3]))])\n\n            if iter_num == epoch_length:\n                loss_rpn_cls = np.mean(losses[:, 0])\n                loss_rpn_regr = np.mean(losses[:, 1])\n                loss_class_cls = np.mean(losses[:, 2])\n                loss_class_regr = np.mean(losses[:, 3])\n                class_acc = np.mean(losses[:, 4])\n\n                mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)\n                rpn_accuracy_for_epoch = []\n\n                if C.verbose:\n                    print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes))\n                    print('Classifier accuracy for bounding boxes from RPN: {}'.format(class_acc))\n                    print('Loss RPN classifier: {}'.format(loss_rpn_cls))\n                    print('Loss RPN regression: {}'.format(loss_rpn_regr))\n                    print('Loss Detector classifier: {}'.format(loss_class_cls))\n                    print('Loss Detector regression: {}'.format(loss_class_regr))\n                    print('Total loss: {}'.format(loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr))\n                    print('Elapsed time: {}'.format(time.time() - start_time))\n                    elapsed_time = (time.time()-start_time)/60\n\n                curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr\n                iter_num = 0\n                start_time = time.time()\n\n                if curr_loss < best_loss:\n                    if C.verbose:\n                        print('Total loss decreased from {} to {}, saving weights'.format(best_loss,curr_loss))\n                    best_loss = curr_loss\n                    model_all.save_weights(C.model_path)\n\n                new_row = {'mean_overlapping_bboxes':round(mean_overlapping_bboxes, 3), \n                           'class_acc':round(class_acc, 3), \n                           'loss_rpn_cls':round(loss_rpn_cls, 3), \n                           'loss_rpn_regr':round(loss_rpn_regr, 3), \n                           'loss_class_cls':round(loss_class_cls, 3), \n                           'loss_class_regr':round(loss_class_regr, 3), \n                           'curr_loss':round(curr_loss, 3), \n                           'elapsed_time':round(elapsed_time, 3), \n                           'mAP': 0}\n\n#                 record_df = record_df.append(new_row, ignore_index=True)\n#                 record_df.to_csv('/kaggle/working/record_df.csv', index=0)\n\n                break\n\n        except Exception as e:\n            print('Exception: {}'.format(e))\n            continue\n\nprint('Training complete, exiting.')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/voccsv/train_image_data.csv')\n# train_df.head()\ntrain_df.drop(\"Unnamed: 0\",axis=1,inplace=True)\ndef creatingPath(x):\n    return (\"../input/voctrain/VOCdevkit/VOC2007/JPEGImages/\"+x)\nperson_df[\"path\"] = person_df[\"image_path\"].apply(creatingPath)\n\nimport cv2\nwith open(\"/kaggle/working/annotation.txt\", \"w+\") as f:\n  for idx, row in person_df.iterrows():\n      img = cv2.imread(row['path'])\n      height, width = img.shape[:2]\n      x1 = int(row['xmin'])\n      x2 = int(row['xmax'])\n      y1 = int(row['ymin'])\n      y2 = int(row['ymax'])\n\n      fileName = row['path']\n      className = row['class_name']\n      f.write(fileName + ',' + str(x1) + ',' + str(y1) + ',' + str(x2) + ',' + str(y2) + ',' + className + '\\n')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}