{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# !pip install --upgrade tensorflow\nimport random\nimport sys\nimport time\nfrom optparse import OptionParser\nimport pickle\nimport math\nimport cv2\nimport copy\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nfrom sklearn.metrics import average_precision_score\nfrom keras import backend as K\nfrom keras.optimizers import Adam, SGD, RMSprop\nfrom keras.layers import Flatten, Dense, Input, Conv2D, MaxPooling2D, Dropout\nfrom keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, TimeDistributed\nfrom keras.engine.topology import get_source_inputs\nfrom keras.utils import layer_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.objectives import categorical_crossentropy\nfrom keras.models import Model\nfrom keras.utils import generic_utils\nfrom keras.engine import Layer, InputSpec\nfrom keras import initializers, regularizers\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport pandas.util.testing as tm\nimport cv2\nimport csv\nimport glob\nimport random\nimport itertools\nfrom collections import Counter\nfrom math import ceil\nfrom tqdm.notebook import tqdm\nfrom IPython.core.interactiveshell import InteractiveShell\nimport scipy.stats as stats\nimport matplotlib as mpl\nimport os\nimport sys\nfrom pprint import pprint\nimport pandas.util.testing as tm\nInteractiveShell.ast_node_interactivity = \"all\"\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=pd.read_csv('/kaggle/input/global-wheat-detection/train.csv')\nprint(\"Initial Shape\",df_train.shape)\ndf_train=df_train[df_train['bbox'].notnull()]\nprint(\"Initial Shape\",df_train.shape)\nprint(\"As shapes are same, we have wheats inside all the images.\")\ntotal_images=df_train['image_id'].nunique()\nimage_names=df_train['image_id'].unique()\n# df_train.groupby(['image_id'])['width'].count().reset_index().rename(columns={'width':'count'}).sort_values(by='count', ascending=False).reset_index(drop=True).to_csv('Image_count.csv')\nbboxes = np.stack(df_train['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x_min', 'y_min', 'width', 'height']):\n    df_train[column] = bboxes[:,i]\ndf_train['x_max']=df_train['x_min']+df_train['width']\ndf_train['y_max']=df_train['y_min']+df_train['height']\ndf_train['bbox_area']=df_train['width']*df_train['height']\ndf_train['in_bbox_class']=1\ndf_train['centroid_x']=df_train['x_min']+df_train['width']/2\ndf_train['centroid_y']=df_train['y_min']+df_train['height']/2\nassert len(df_train['x_max']) == df_train.shape[0]\nassert len(df_train['y_max']) == df_train.shape[0]\ndf_train['image_id']=df_train['image_id'].map('/kaggle/input/global-wheat-detection/train/{}.jpg'.format)\ndf_train['image_id']=df_train['image_id'].astype('str')\ndf_train.drop(columns=['bbox'], inplace=True)\n# df_train.to_csv('modified_train.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['bbox_area'].value_counts().hist(bins=10)\nplt.savefig('Bbox_Area_plot')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['source'].value_counts().plot(kind='bar')\nplt.grid('on',which='both')\nplt.title('Source counts')\nplt.savefig('Source Counts Images')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns_plot=sns.boxplot(x='centroid_x',data=df_train)\nplt.title('X-Pixel')\nplt.savefig('centroid_x_box_plot')\nplt.show()\nsns_plot=sns.boxplot(x='centroid_y',data=df_train)\nplt.title('Y-Pixel')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"it = stats.norm.pdf(df_train['centroid_x'], df_train['centroid_x'].mean(), df_train['centroid_x'].std()) \nmpl.rcParams['agg.path.chunksize'] = 10000\nplt.plot(df_train['centroid_x'],it,'-o')\nplt.title('Normal Distribution of Centroid X')\nplt.savefig('Normally_Distributed_Cordinates')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"it = stats.norm.pdf(df_train['centroid_y'], df_train['centroid_y'].mean(), df_train['centroid_y'].std()) \nmpl.rcParams['agg.path.chunksize'] = 10000\nplt.plot(df_train['centroid_y'],it,'-o')\nplt.title('Normal Distribution of Centroid Y')\nplt.savefig('Normally_Distributed_Cordinates_Y')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check data anamolies","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"condition=((df_train[\"x_max\"] > 1024.0) & (df_train[\"y_max\"] > 1024.0) & (df_train[\"x_min\"] < 0) & (df_train[\"x_max\"] < 0))\nif len(df_train[condition])==0:\n    print(\"Train data doesn't has values greate than 1024 or less than 0\")\nelse:\n    print(\"x_max is fine\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert len(glob.glob('/kaggle/input/global-wheat-detection/train/*')) == len([i for i in glob.glob('/kaggle/input/global-wheat-detection/train/*') if i.split('.')[1]=='jpg']) , \"All are not JPGs\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_image_size(image_folder_path):\n    total_img_list = glob.glob(os.path.join(image_folder_path,\"*\"))\n    counter = 0\n    for image in tqdm(total_img_list, desc = \"Checking in progress\"):\n        try:\n            img = cv2.imread(image)\n            height, width = img.shape[1], img.shape[0]\n            if not (height == 1024 and width == 1024):\n                counter = counter + 1\n        except:\n            print(\"This {} is problematic.\".format(image))\n    return counter ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validating images in the directory","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_dict=set(df_train['image_id'])\nlist_of_extra_folder_files=[]\nfolder_file_list=[]\nfor file in glob.glob('/kaggle/input/global-wheat-detection/train/*'):\n    folder_file_list.append(file)\nprint(f\"No of extra Jpegs in folder is\", len(set(folder_file_list)-set(image_dict)))\nprint(\"Following are the exta files in the directory: \")\npprint( [os.path.basename(fl) for fl in set(folder_file_list)-set(image_dict)], indent =4, width=120, compact=True)\nextra_files=set(folder_file_list)-set(image_dict)\nprint('Extra files with annotations are',len(set(image_dict)-set(folder_file_list)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make a label to classify data based on availability.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_img(img_matrix_list, title_list,nrows, ncols, main_title=\"\"):\n    fig, myaxes = plt.subplots(figsize=(20, 15), nrows=nrows, ncols=ncols, squeeze=False)\n    fig.suptitle(main_title, fontsize = 30)\n    fig.subplots_adjust(wspace=0.3)\n    fig.subplots_adjust(hspace=0.3)\n    for i, (img, title) in enumerate(zip(img_matrix_list, title_list)):\n        myaxes[i // ncols][i % ncols].imshow(img)\n        myaxes[i // ncols][i % ncols].set_title(os.path.basename(title), fontsize=15)\n#     plt.savefig(main_title)\n    plt.show()\ndef draw_rect(img, bboxes, color=None):\n    img = img.copy()\n    bboxes = bboxes[:, :4]\n    bboxes = bboxes.reshape(-1, 4)\n    for bbox in bboxes:\n        pt1, pt2 = (bbox[0], bbox[1]), (bbox[2], bbox[3])\n        pt1 = int(pt1[0]), int(pt1[1])\n        pt2 = int(pt2[0]), int(pt2[1])\n        img = cv2.rectangle(img.copy(), pt1, pt2, color, int(max(img.shape[:2]) / 200))\n    return img\ndef draw_random_image_and_box(img_files_list):\n    bbox_list = []\n    img_matrix_list = []\n    random_image_matrix_list = []\n    for img_file in img_files_list:\n        image_file_path = os.path.join(img_file)\n        img = cv2.imread(image_file_path)[:,:,::-1] \n        height, width, channels = img.shape\n        _df=df_train[df_train['image_id']==img_file]\n        bbox_list.append(_df.values[:,4:8].tolist())\n        img_matrix_list.append(img)\n    plot_img(img_matrix_list,img_files_list,3,4,\"Random Wheat Pictures\")\n    final_bbox_list = []\n    for bboxes, img in zip(bbox_list, img_matrix_list):\n        final_bbox_array = np.array([])\n        for bbox in bboxes:\n            bbox = np.array(bbox).reshape(1,4)\n            final_bbox_array = np.append(final_bbox_array, bbox)\n        final_bbox_array = final_bbox_array.reshape(-1,4)\n        random_image = draw_rect(img.copy(), final_bbox_array.copy(),color = (255,0,0))\n        random_image_matrix_list.append(random_image)\n    plot_img(random_image_matrix_list, img_files_list,3 ,4, main_title=\"Bounding Box Wheat Images\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_files_list = np.random.choice(list(image_dict),12)\ndraw_random_image_and_box(img_files_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying Faster - RCNN: Credit to the training code goes to **RockyXu66 Git Repository**.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Config setting**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n\tdef __init__(self):\n\t\t# Print the process or not\n\t\tself.verbose = True\n\t\t# Name of base network\n\t\tself.network = 'vgg'\n\t\t# Setting for data augmentation\n\t\tself.use_horizontal_flips = False\n\t\tself.use_vertical_flips = False\n\t\tself.rot_90 = False\n\t\t# Anchor box scales\n    # Note that if im_size is smaller, anchor_box_scales should be scaled\n    # Original anchor_box_scales in the paper is [128, 256, 512]\n\t\tself.anchor_box_scales = [64, 128, 256] \n\t\t# Anchor box ratios\n\t\tself.anchor_box_ratios = [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]\n\t\t# Size to resize the smallest side of the image\n\t\t# Original setting in paper is 600. Set to 300 in here to save training time\n\t\tself.im_size = 300\n\t\t# image channel-wise mean to subtract\n\t\tself.img_channel_mean = [103.939, 116.779, 123.68]\n\t\tself.img_scaling_factor = 1.0\n\t\t# number of ROIs at once\n\t\tself.num_rois = 4\n\t\t# stride at the RPN (this depends on the network configuration)\n\t\tself.rpn_stride = 16\n\t\tself.balanced_classes = False\n\t\t# scaling the stdev\n\t\tself.std_scaling = 4.0\n\t\tself.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]\n\t\t# overlaps for RPN\n\t\tself.rpn_min_overlap = 0.3\n\t\tself.rpn_max_overlap = 0.7\n\t\t# overlaps for classifier ROIs\n\t\tself.classifier_min_overlap = 0.1\n\t\tself.classifier_max_overlap = 0.5\n\t\t# placeholder for the class mapping, automatically generated by the parser\n\t\tself.class_mapping = None\n\t\tself.model_path = None\nclass RoiPoolingConv(Layer):\n    '''ROI pooling layer for 2D inputs.\n    See Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition,\n    K. He, X. Zhang, S. Ren, J. Sun\n    # Arguments\n        pool_size: int\n            Size of pooling region to use. pool_size = 7 will result in a 7x7 region.\n        num_rois: number of regions of interest to be used\n    # Input shape\n        list of two 4D tensors [X_img,X_roi] with shape:\n        X_img:\n        `(1, rows, cols, channels)`\n        X_roi:\n        `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n    # Output shape\n        3D tensor with shape:\n        `(1, num_rois, channels, pool_size, pool_size)`\n    '''\n    def __init__(self, pool_size, num_rois, **kwargs):\n\n        self.dim_ordering = K.image_data_format()\n        self.pool_size = pool_size\n        self.num_rois = num_rois\n\n        super(RoiPoolingConv, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.nb_channels = input_shape[0][3]   \n\n    def compute_output_shape(self, input_shape):\n        return None, self.num_rois, self.pool_size, self.pool_size, self.nb_channels\n\n    def call(self, x, mask=None):\n\n        assert(len(x) == 2)\n\n        # x[0] is image with shape (rows, cols, channels)\n        img = x[0]\n\n        # x[1] is roi with shape (num_rois,4) with ordering (x,y,w,h)\n        rois = x[1]\n\n        input_shape = K.shape(img)\n\n        outputs = []\n\n        for roi_idx in range(self.num_rois):\n\n            x = rois[0, roi_idx, 0]\n            y = rois[0, roi_idx, 1]\n            w = rois[0, roi_idx, 2]\n            h = rois[0, roi_idx, 3]\n\n            x = K.cast(x, 'int32')\n            y = K.cast(y, 'int32')\n            w = K.cast(w, 'int32')\n            h = K.cast(h, 'int32')\n\n            # Resized roi of the image to pooling size (7x7)\n            rs = tf.image.resize(img[:, y:y+h, x:x+w, :], (self.pool_size, self.pool_size))\n            outputs.append(rs)\n                \n\n        final_output = K.concatenate(outputs, axis=0)\n\n        # Reshape to (1, num_rois, pool_size, pool_size, nb_channels)\n        # Might be (1, 4, 7, 7, 3)\n        final_output = K.reshape(final_output, (1, self.num_rois, self.pool_size, self.pool_size, self.nb_channels))\n\n        # permute_dimensions is similar to transpose\n        final_output = K.permute_dimensions(final_output, (0, 1, 2, 3, 4))\n\n        return final_output\n    def get_config(self):\n        config = {'pool_size': self.pool_size,\n                  'num_rois': self.num_rois}\n        base_config = super(RoiPoolingConv, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\ndef get_data(input_path):\n    \"\"\"Parse the data from annotation file\n    Args:\n        input_path: annotation file path\n\n    Returns:\n        all_data: list(filepath, width, height, list(bboxes))\n        classes_count: dict{key:class_name, value:count_num} \n            e.g. {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745}\n        class_mapping: dict{key:class_name, value: idx}\n            e.g. {'Car': 0, 'Mobile phone': 1, 'Person': 2}\n    \"\"\"\n    found_bg = False\n    all_imgs = {}\n\n    classes_count = {}\n\n    class_mapping = {}\n\n    visualise = True\n\n    i = 1\n\n    with open(input_path,'r') as f:\n\n#         print('Parsing annotation files')\n\n        for line in f:\n\n            # Print process\n            sys.stdout.write('\\r'+'idx=' + str(i))\n            i += 1\n\n            line_split = line.strip().split(',')\n\n            # Make sure the info saved in annotation file matching the format (path_filename, x1, y1, x2, y2, class_name)\n            # Note:\n            #\tOne path_filename might has several classes (class_name)\n            #\tx1, y1, x2, y2 are the pixel value of the origial image, not the ratio value\n            #\t(x1, y1) top left coordinates; (x2, y2) bottom right coordinates\n            #   x1,y1-------------------\n            #\t|\t\t\t\t\t\t|\n            #\t|\t\t\t\t\t\t|\n            #\t|\t\t\t\t\t\t|\n            #\t|\t\t\t\t\t\t|\n            #\t---------------------x2,y2\n\n            (filename,x1,y1,x2,y2,class_name) = line_split\n\n            if class_name not in classes_count:\n                classes_count[class_name] = 1\n            else:\n                classes_count[class_name] += 1\n\n            if class_name not in class_mapping:\n                if class_name == 'bg' and found_bg == False:\n                    print('Found class name with special name bg. Will be treated as a background region (this is usually for hard negative mining).')\n                    found_bg = True\n                class_mapping[class_name] = len(class_mapping)\n\n            if filename not in all_imgs:\n                all_imgs[filename] = {}\n\n                img = cv2.imread(filename)\n                (rows,cols) = img.shape[:2]\n                all_imgs[filename]['filepath'] = filename\n                all_imgs[filename]['width'] = cols\n                all_imgs[filename]['height'] = rows\n                all_imgs[filename]['bboxes'] = []\n                # if np.random.randint(0,6) > 0:\n                # \tall_imgs[filename]['imageset'] = 'trainval'\n                # else:\n                # \tall_imgs[filename]['imageset'] = 'test'\n\n            all_imgs[filename]['bboxes'].append({'class': class_name, 'x1': int(x1), 'x2': int(x2), 'y1': int(y1), 'y2': int(y2)})\n        all_data = []\n        for key in all_imgs:\n            all_data.append(all_imgs[key])\n        # make sure the bg class is last in the list\n        if found_bg:\n            if class_mapping['bg'] != len(class_mapping) - 1:\n                key_to_switch = [key for key in class_mapping.keys() if class_mapping[key] == len(class_mapping)-1][0]\n                val_to_switch = class_mapping['bg']\n                class_mapping['bg'] = len(class_mapping) - 1\n                class_mapping[key_to_switch] = val_to_switch\n        return all_data, classes_count, class_mapping\ndef get_img_output_length(width, height):\n    def get_output_length(input_length):\n        return input_length//16\n\n    return get_output_length(width), get_output_length(height)    \n\ndef nn_base(input_tensor=None, trainable=False):\n\n\n    input_shape = (None, None, 3)\n\n    if input_tensor is None:\n        img_input = Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n\n    bn_axis = 3\n\n    # Block 1\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)\n\n    # Block 2\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)\n\n    # Block 3\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)\n\n    # Block 4\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)\n\n    # Block 5\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)\n    # x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)\n\n    return x\ndef rpn_layer(base_layers, num_anchors):\n    \"\"\"Create a rpn layer\n        Step1: Pass through the feature map from base layer to a 3x3 512 channels convolutional layer\n                Keep the padding 'same' to preserve the feature map's size\n        Step2: Pass the step1 to two (1,1) convolutional layer to replace the fully connected layer\n                classification layer: num_anchors (9 in here) channels for 0, 1 sigmoid activation output\n                regression layer: num_anchors*4 (36 in here) channels for computing the regression of bboxes with linear activation\n    Args:\n        base_layers: vgg in here\n        num_anchors: 9 in here\n\n    Returns:\n        [x_class, x_regr, base_layers]\n        x_class: classification for whether it's an object\n        x_regr: bboxes regression\n        base_layers: vgg in here\n    \"\"\"\n    x = Conv2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)\n\n    x_class = Conv2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)\n    x_regr = Conv2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)\n\n    return [x_class, x_regr, base_layers]\ndef classifier_layer(base_layers, input_rois, num_rois, nb_classes = 4):\n    \"\"\"Create a classifier layer\n    \n    Args:\n        base_layers: vgg\n        input_rois: `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n        num_rois: number of rois to be processed in one time (4 in here)\n\n    Returns:\n        list(out_class, out_regr)\n        out_class: classifier layer output\n        out_regr: regression layer output\n    \"\"\"\n\n    input_shape = (num_rois,7,7,512)\n\n    pooling_regions = 7\n\n    # out_roi_pool.shape = (1, num_rois, channels, pool_size, pool_size)\n    # num_rois (4) 7x7 roi pooling\n    out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois])\n\n    # Flatten the convlutional layer and connected to 2 FC and 2 dropout\n    out = TimeDistributed(Flatten(name='flatten'))(out_roi_pool)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc1'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc2'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n\n    # There are two output layer\n    # out_class: softmax acivation function for classify the class name of the object\n    # out_regr: linear activation function for bboxes coordinates regression\n    out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out)\n    # note: no regression target for bg class\n    out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out)\n\n    return [out_class, out_regr]\ndef union(au, bu, area_intersection):\n\tarea_a = (au[2] - au[0]) * (au[3] - au[1])\n\tarea_b = (bu[2] - bu[0]) * (bu[3] - bu[1])\n\tarea_union = area_a + area_b - area_intersection\n\treturn area_union\n\n\ndef intersection(ai, bi):\n\tx = max(ai[0], bi[0])\n\ty = max(ai[1], bi[1])\n\tw = min(ai[2], bi[2]) - x\n\th = min(ai[3], bi[3]) - y\n\tif w < 0 or h < 0:\n\t\treturn 0\n\treturn w*h\n\n\ndef iou(a, b):\n\t# a and b should be (x1,y1,x2,y2)\n\n\tif a[0] >= a[2] or a[1] >= a[3] or b[0] >= b[2] or b[1] >= b[3]:\n\t\treturn 0.0\n\n\tarea_i = intersection(a, b)\n\tarea_u = union(a, b, area_i)\n\n\treturn float(area_i) / float(area_u + 1e-6)\ndef calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function):\n\t\"\"\"(Important part!) Calculate the rpn for all anchors \n\t\tIf feature map has shape 38x50=1900, there are 1900x9=17100 potential anchors\n\t\n\tArgs:\n\t\tC: config\n\t\timg_data: augmented image data\n\t\twidth: original image width (e.g. 600)\n\t\theight: original image height (e.g. 800)\n\t\tresized_width: resized image width according to C.im_size (e.g. 300)\n\t\tresized_height: resized image height according to C.im_size (e.g. 400)\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\n\tReturns:\n\t\ty_rpn_cls: list(num_bboxes, y_is_box_valid + y_rpn_overlap)\n\t\t\ty_is_box_valid: 0 or 1 (0 means the box is invalid, 1 means the box is valid)\n\t\t\ty_rpn_overlap: 0 or 1 (0 means the box is not an object, 1 means the box is an object)\n\t\ty_rpn_regr: list(num_bboxes, 4*y_rpn_overlap + y_rpn_regr)\n\t\t\ty_rpn_regr: x1,y1,x2,y2 bunding boxes coordinates\n\t\"\"\"\n\tdownscale = float(C.rpn_stride) \n\tanchor_sizes = C.anchor_box_scales   # 128, 256, 512\n\tanchor_ratios = C.anchor_box_ratios  # 1:1, 1:2*sqrt(2), 2*sqrt(2):1\n\tnum_anchors = len(anchor_sizes) * len(anchor_ratios) # 3x3=9\n\n\t# calculate the output map size based on the network architecture\n\t(output_width, output_height) = img_length_calc_function(resized_width, resized_height)\n\n\tn_anchratios = len(anchor_ratios)    # 3\n\t\n\t# initialise empty output objectives\n\ty_rpn_overlap = np.zeros((output_height, output_width, num_anchors))\n\ty_is_box_valid = np.zeros((output_height, output_width, num_anchors))\n\ty_rpn_regr = np.zeros((output_height, output_width, num_anchors * 4))\n\n\tnum_bboxes = len(img_data['bboxes'])\n\n\tnum_anchors_for_bbox = np.zeros(num_bboxes).astype(int)\n\tbest_anchor_for_bbox = -1*np.ones((num_bboxes, 4)).astype(int)\n\tbest_iou_for_bbox = np.zeros(num_bboxes).astype(np.float32)\n\tbest_x_for_bbox = np.zeros((num_bboxes, 4)).astype(int)\n\tbest_dx_for_bbox = np.zeros((num_bboxes, 4)).astype(np.float32)\n\n\t# get the GT box coordinates, and resize to account for image resizing\n\tgta = np.zeros((num_bboxes, 4))\n\tfor bbox_num, bbox in enumerate(img_data['bboxes']):\n\t\t# get the GT box coordinates, and resize to account for image resizing\n\t\tgta[bbox_num, 0] = bbox['x1'] * (resized_width / float(width))\n\t\tgta[bbox_num, 1] = bbox['x2'] * (resized_width / float(width))\n\t\tgta[bbox_num, 2] = bbox['y1'] * (resized_height / float(height))\n\t\tgta[bbox_num, 3] = bbox['y2'] * (resized_height / float(height))\n\t\n\t# rpn ground truth\n\n\tfor anchor_size_idx in range(len(anchor_sizes)):\n\t\tfor anchor_ratio_idx in range(n_anchratios):\n\t\t\tanchor_x = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][0]\n\t\t\tanchor_y = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][1]\t\n\t\t\t\n\t\t\tfor ix in range(output_width):\t\t\t\t\t\n\t\t\t\t# x-coordinates of the current anchor box\t\n\t\t\t\tx1_anc = downscale * (ix + 0.5) - anchor_x / 2\n\t\t\t\tx2_anc = downscale * (ix + 0.5) + anchor_x / 2\t\n\t\t\t\t\n\t\t\t\t# ignore boxes that go across image boundaries\t\t\t\t\t\n\t\t\t\tif x1_anc < 0 or x2_anc > resized_width:\n\t\t\t\t\tcontinue\n\t\t\t\t\t\n\t\t\t\tfor jy in range(output_height):\n\n\t\t\t\t\t# y-coordinates of the current anchor box\n\t\t\t\t\ty1_anc = downscale * (jy + 0.5) - anchor_y / 2\n\t\t\t\t\ty2_anc = downscale * (jy + 0.5) + anchor_y / 2\n\n\t\t\t\t\t# ignore boxes that go across image boundaries\n\t\t\t\t\tif y1_anc < 0 or y2_anc > resized_height:\n\t\t\t\t\t\tcontinue\n\n\t\t\t\t\t# bbox_type indicates whether an anchor should be a target\n\t\t\t\t\t# Initialize with 'negative'\n\t\t\t\t\tbbox_type = 'neg'\n\n\t\t\t\t\t# this is the best IOU for the (x,y) coord and the current anchor\n\t\t\t\t\t# note that this is different from the best IOU for a GT bbox\n\t\t\t\t\tbest_iou_for_loc = 0.0\n\n\t\t\t\t\tfor bbox_num in range(num_bboxes):\n\t\t\t\t\t\t\n\t\t\t\t\t\t# get IOU of the current GT box and the current anchor box\n\t\t\t\t\t\tcurr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1_anc, y1_anc, x2_anc, y2_anc])\n\t\t\t\t\t\t# calculate the regression targets if they will be needed\n\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num] or curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\tcx = (gta[bbox_num, 0] + gta[bbox_num, 1]) / 2.0\n\t\t\t\t\t\t\tcy = (gta[bbox_num, 2] + gta[bbox_num, 3]) / 2.0\n\t\t\t\t\t\t\tcxa = (x1_anc + x2_anc)/2.0\n\t\t\t\t\t\t\tcya = (y1_anc + y2_anc)/2.0\n\n\t\t\t\t\t\t\t# x,y are the center point of ground-truth bbox\n\t\t\t\t\t\t\t# xa,ya are the center point of anchor bbox (xa=downscale * (ix + 0.5); ya=downscale * (iy+0.5))\n\t\t\t\t\t\t\t# w,h are the width and height of ground-truth bbox\n\t\t\t\t\t\t\t# wa,ha are the width and height of anchor bboxe\n\t\t\t\t\t\t\t# tx = (x - xa) / wa\n\t\t\t\t\t\t\t# ty = (y - ya) / ha\n\t\t\t\t\t\t\t# tw = log(w / wa)\n\t\t\t\t\t\t\t# th = log(h / ha)\n\t\t\t\t\t\t\ttx = (cx - cxa) / (x2_anc - x1_anc)\n\t\t\t\t\t\t\tty = (cy - cya) / (y2_anc - y1_anc)\n\t\t\t\t\t\t\ttw = np.log((gta[bbox_num, 1] - gta[bbox_num, 0]) / (x2_anc - x1_anc))\n\t\t\t\t\t\t\tth = np.log((gta[bbox_num, 3] - gta[bbox_num, 2]) / (y2_anc - y1_anc))\n\t\t\t\t\t\t\n\t\t\t\t\t\tif img_data['bboxes'][bbox_num]['class'] != 'bg':\n\n\t\t\t\t\t\t\t# all GT boxes should be mapped to an anchor box, so we keep track of which anchor box was best\n\t\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num]:\n\t\t\t\t\t\t\t\tbest_anchor_for_bbox[bbox_num] = [jy, ix, anchor_ratio_idx, anchor_size_idx]\n\t\t\t\t\t\t\t\tbest_iou_for_bbox[bbox_num] = curr_iou\n\t\t\t\t\t\t\t\tbest_x_for_bbox[bbox_num,:] = [x1_anc, x2_anc, y1_anc, y2_anc]\n\t\t\t\t\t\t\t\tbest_dx_for_bbox[bbox_num,:] = [tx, ty, tw, th]\n\n\t\t\t\t\t\t\t# we set the anchor to positive if the IOU is >0.7 (it does not matter if there was another better box, it just indicates overlap)\n\t\t\t\t\t\t\tif curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\t\tbbox_type = 'pos'\n\t\t\t\t\t\t\t\tnum_anchors_for_bbox[bbox_num] += 1\n\t\t\t\t\t\t\t\t# we update the regression layer target if this IOU is the best for the current (x,y) and anchor position\n\t\t\t\t\t\t\t\tif curr_iou > best_iou_for_loc:\n\t\t\t\t\t\t\t\t\tbest_iou_for_loc = curr_iou\n\t\t\t\t\t\t\t\t\tbest_regr = (tx, ty, tw, th)\n\n\t\t\t\t\t\t\t# if the IOU is >0.3 and <0.7, it is ambiguous and no included in the objective\n\t\t\t\t\t\t\tif C.rpn_min_overlap < curr_iou < C.rpn_max_overlap:\n\t\t\t\t\t\t\t\t# gray zone between neg and pos\n\t\t\t\t\t\t\t\tif bbox_type != 'pos':\n\t\t\t\t\t\t\t\t\tbbox_type = 'neutral'\n\n\t\t\t\t\t# turn on or off outputs depending on IOUs\n\t\t\t\t\tif bbox_type == 'neg':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'neutral':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'pos':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\tstart = 4 * (anchor_ratio_idx + n_anchratios * anchor_size_idx)\n\t\t\t\t\t\ty_rpn_regr[jy, ix, start:start+4] = best_regr\n\n\t# we ensure that every bbox has at least one positive RPN region\n\n\tfor idx in range(num_anchors_for_bbox.shape[0]):\n\t\tif num_anchors_for_bbox[idx] == 0:\n\t\t\t# no box with an IOU greater than zero ...\n\t\t\tif best_anchor_for_bbox[idx, 0] == -1:\n\t\t\t\tcontinue\n\t\t\ty_is_box_valid[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\ty_rpn_overlap[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\tstart = 4 * (best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3])\n\t\t\ty_rpn_regr[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], start:start+4] = best_dx_for_bbox[idx, :]\n\n\ty_rpn_overlap = np.transpose(y_rpn_overlap, (2, 0, 1))\n\ty_rpn_overlap = np.expand_dims(y_rpn_overlap, axis=0)\n\n\ty_is_box_valid = np.transpose(y_is_box_valid, (2, 0, 1))\n\ty_is_box_valid = np.expand_dims(y_is_box_valid, axis=0)\n\n\ty_rpn_regr = np.transpose(y_rpn_regr, (2, 0, 1))\n\ty_rpn_regr = np.expand_dims(y_rpn_regr, axis=0)\n\n\tpos_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1))\n\tneg_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 0, y_is_box_valid[0, :, :, :] == 1))\n\n\tnum_pos = len(pos_locs[0])\n\n\t# one issue is that the RPN has many more negative than positive regions, so we turn off some of the negative\n\t# regions. We also limit it to 256 regions.\n\tnum_regions = 256\n\n\tif len(pos_locs[0]) > num_regions/2:\n\t\tval_locs = random.sample(range(len(pos_locs[0])), len(pos_locs[0]) - num_regions/2)\n\t\ty_is_box_valid[0, pos_locs[0][val_locs], pos_locs[1][val_locs], pos_locs[2][val_locs]] = 0\n\t\tnum_pos = num_regions/2\n\n\tif len(neg_locs[0]) + num_pos > num_regions:\n\t\tval_locs = random.sample(range(len(neg_locs[0])), len(neg_locs[0]) - num_pos)\n\t\ty_is_box_valid[0, neg_locs[0][val_locs], neg_locs[1][val_locs], neg_locs[2][val_locs]] = 0\n\n\ty_rpn_cls = np.concatenate([y_is_box_valid, y_rpn_overlap], axis=1)\n\ty_rpn_regr = np.concatenate([np.repeat(y_rpn_overlap, 4, axis=1), y_rpn_regr], axis=1)\n\n\treturn np.copy(y_rpn_cls), np.copy(y_rpn_regr), num_pos\ndef get_new_img_size(width, height, img_min_side=300):\n\tif width <= height:\n\t\tf = float(img_min_side) / width\n\t\tresized_height = int(f * height)\n\t\tresized_width = img_min_side\n\telse:\n\t\tf = float(img_min_side) / height\n\t\tresized_width = int(f * width)\n\t\tresized_height = img_min_side\n\n\treturn resized_width, resized_height\n\ndef augment(img_data, config, augment=True):\n\tassert 'filepath' in img_data\n\tassert 'bboxes' in img_data\n\tassert 'width' in img_data\n\tassert 'height' in img_data\n\n\timg_data_aug = copy.deepcopy(img_data)\n\n\timg = cv2.imread(img_data_aug['filepath'])\n\n\tif augment:\n\t\trows, cols = img.shape[:2]\n\n\t\tif config.use_horizontal_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 1)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\tbbox['x1'] = cols - x2\n\n\t\tif config.use_vertical_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 0)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\tbbox['y1'] = rows - y2\n\n\t\tif config.rot_90:\n\t\t\tangle = np.random.choice([0,90,180,270],1)[0]\n\t\t\tif angle == 270:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 0)\n\t\t\telif angle == 180:\n\t\t\t\timg = cv2.flip(img, -1)\n\t\t\telif angle == 90:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 1)\n\t\t\telif angle == 0:\n\t\t\t\tpass\n\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tif angle == 270:\n\t\t\t\t\tbbox['x1'] = y1\n\t\t\t\t\tbbox['x2'] = y2\n\t\t\t\t\tbbox['y1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = cols - x1\n\t\t\t\telif angle == 180:\n\t\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\t\tbbox['x1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = rows - y2\n\t\t\t\telif angle == 90:\n\t\t\t\t\tbbox['x1'] = rows - y2\n\t\t\t\t\tbbox['x2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = x1\n\t\t\t\t\tbbox['y2'] = x2        \n\t\t\t\telif angle == 0:\n\t\t\t\t\tpass\n\n\timg_data_aug['width'] = img.shape[1]\n\timg_data_aug['height'] = img.shape[0]\n\treturn img_data_aug, img\ndef get_anchor_gt(all_img_data, C, img_length_calc_function, mode='train'):\n\t\"\"\" Yield the ground-truth anchors as Y (labels)\n\t\t\n\tArgs:\n\t\tall_img_data: list(filepath, width, height, list(bboxes))\n\t\tC: config\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\t\tmode: 'train' or 'test'; 'train' mode need augmentation\n\n\tReturns:\n\t\tx_img: image data after resized and scaling (smallest size = 300px)\n\t\tY: [y_rpn_cls, y_rpn_regr]\n\t\timg_data_aug: augmented image data (original image with augmentation)\n\t\tdebug_img: show image for debug\n\t\tnum_pos: show number of positive anchors for debug\n\t\"\"\"\n\twhile True:\n\n\t\tfor img_data in all_img_data:\n\t\t\ttry:\n\n\t\t\t\t# read in image, and optionally add augmentation\n\n\t\t\t\tif mode == 'train':\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=True)\n\t\t\t\telse:\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=False)\n\n\t\t\t\t(width, height) = (img_data_aug['width'], img_data_aug['height'])\n\t\t\t\t(rows, cols, _) = x_img.shape\n\n\t\t\t\tassert cols == width\n\t\t\t\tassert rows == height\n\n\t\t\t\t# get image dimensions for resizing\n\t\t\t\t(resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n\t\t\t\t# resize the image so that smalles side is length = 300px\n\t\t\t\tx_img = cv2.resize(x_img, (resized_width, resized_height), interpolation=cv2.INTER_CUBIC)\n\t\t\t\tdebug_img = x_img.copy()\n\n\t\t\t\ttry:\n\t\t\t\t\ty_rpn_cls, y_rpn_regr, num_pos = calc_rpn(C, img_data_aug, width, height, resized_width, resized_height, img_length_calc_function)\n\t\t\t\texcept:\n\t\t\t\t\tcontinue\n\n\t\t\t\t# Zero-center by mean pixel, and preprocess image\n\n\t\t\t\tx_img = x_img[:,:, (2, 1, 0)]  # BGR -> RGB\n\t\t\t\tx_img = x_img.astype(np.float32)\n\t\t\t\tx_img[:, :, 0] -= C.img_channel_mean[0]\n\t\t\t\tx_img[:, :, 1] -= C.img_channel_mean[1]\n\t\t\t\tx_img[:, :, 2] -= C.img_channel_mean[2]\n\t\t\t\tx_img /= C.img_scaling_factor\n\n\t\t\t\tx_img = np.transpose(x_img, (2, 0, 1))\n\t\t\t\tx_img = np.expand_dims(x_img, axis=0)\n\n\t\t\t\ty_rpn_regr[:, y_rpn_regr.shape[1]//2:, :, :] *= C.std_scaling\n\n\t\t\t\tx_img = np.transpose(x_img, (0, 2, 3, 1))\n\t\t\t\ty_rpn_cls = np.transpose(y_rpn_cls, (0, 2, 3, 1))\n\t\t\t\ty_rpn_regr = np.transpose(y_rpn_regr, (0, 2, 3, 1))\n\n\t\t\t\tyield np.copy(x_img), [np.copy(y_rpn_cls), np.copy(y_rpn_regr)], img_data_aug, debug_img, num_pos\n\n\t\t\texcept Exception as e:\n\t\t\t\tprint(e)\n\t\t\t\tcontinue\nlambda_rpn_regr = 1.0\nlambda_rpn_class = 1.0\nlambda_cls_regr = 1.0\nlambda_cls_class = 1.0\nepsilon = 1e-4\ndef rpn_loss_regr(num_anchors):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def rpn_loss_regr_fixed_num(y_true, y_pred):\n\n        # x is the difference between true value and predicted vaue\n        x = y_true[:, :, :, 4 * num_anchors:] - y_pred\n\n        # absolute value of x\n        x_abs = K.abs(x)\n\n        # If x_abs <= 1.0, x_bool = 1\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), tf.float32)\n\n        return lambda_rpn_regr * K.sum(\n            y_true[:, :, :, :4 * num_anchors] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :, :4 * num_anchors])\n\n    return rpn_loss_regr_fixed_num\n\n\ndef rpn_loss_cls(num_anchors):\n    \"\"\"Loss function for rpn classification\n    Args:\n        num_anchors: number of anchors (9 in here)\n        y_true[:, :, :, :9]: [0,1,0,0,0,0,0,1,0] means only the second and the eighth box is valid which contains pos or neg anchor => isValid\n        y_true[:, :, :, 9:]: [0,1,0,0,0,0,0,0,0] means the second box is pos and eighth box is negative\n    Returns:\n        lambda * sum((binary_crossentropy(isValid*y_pred,y_true))) / N\n    \"\"\"\n    def rpn_loss_cls_fixed_num(y_true, y_pred):\n\n            return lambda_rpn_class * K.sum(y_true[:, :, :, :num_anchors] * K.binary_crossentropy(y_pred[:, :, :, :], y_true[:, :, :, num_anchors:])) / K.sum(epsilon + y_true[:, :, :, :num_anchors])\n\n    return rpn_loss_cls_fixed_num\n\n\ndef class_loss_regr(num_classes):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def class_loss_regr_fixed_num(y_true, y_pred):\n        x = y_true[:, :, 4*num_classes:] - y_pred\n        x_abs = K.abs(x)\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), 'float32')\n        return lambda_cls_regr * K.sum(y_true[:, :, :4*num_classes] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :4*num_classes])\n    return class_loss_regr_fixed_num\n\n\ndef class_loss_cls(y_true, y_pred):\n    return lambda_cls_class * K.mean(categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))\ndef non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):\n    # code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/\n    # if there are no boxes, return an empty list\n\n    # Process explanation:\n    #   Step 1: Sort the probs list\n    #   Step 2: Find the larget prob 'Last' in the list and save it to the pick list\n    #   Step 3: Calculate the IoU with 'Last' box and other boxes in the list. If the IoU is larger than overlap_threshold, delete the box from list\n    #   Step 4: Repeat step 2 and step 3 until there is no item in the probs list \n    if len(boxes) == 0:\n        return []\n\n    # grab the coordinates of the bounding boxes\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    np.testing.assert_array_less(x1, x2)\n    np.testing.assert_array_less(y1, y2)\n\n    # if the bounding boxes integers, convert them to floats --\n    # this is important since we'll be doing a bunch of divisions\n    if boxes.dtype.kind == \"i\":\n        boxes = boxes.astype(\"float\")\n\n    # initialize the list of picked indexes\t\n    pick = []\n\n    # calculate the areas\n    area = (x2 - x1) * (y2 - y1)\n\n    # sort the bounding boxes \n    idxs = np.argsort(probs)\n\n    # keep looping while some indexes still remain in the indexes\n    # list\n    while len(idxs) > 0:\n        # grab the last index in the indexes list and add the\n        # index value to the list of picked indexes\n        last = len(idxs) - 1\n        i = idxs[last]\n        pick.append(i)\n\n        # find the intersection\n\n        xx1_int = np.maximum(x1[i], x1[idxs[:last]])\n        yy1_int = np.maximum(y1[i], y1[idxs[:last]])\n        xx2_int = np.minimum(x2[i], x2[idxs[:last]])\n        yy2_int = np.minimum(y2[i], y2[idxs[:last]])\n\n        ww_int = np.maximum(0, xx2_int - xx1_int)\n        hh_int = np.maximum(0, yy2_int - yy1_int)\n\n        area_int = ww_int * hh_int\n\n        # find the union\n        area_union = area[i] + area[idxs[:last]] - area_int\n\n        # compute the ratio of overlap\n        overlap = area_int/(area_union + 1e-6)\n\n        # delete all indexes from the index list that have\n        idxs = np.delete(idxs, np.concatenate(([last],\n            np.where(overlap > overlap_thresh)[0])))\n\n        if len(pick) >= max_boxes:\n            break\n\n    # return only the bounding boxes that were picked using the integer data type\n    boxes = boxes[pick].astype(\"int\")\n    probs = probs[pick]\n    return boxes, probs\n\ndef apply_regr_np(X, T):\n    \"\"\"Apply regression layer to all anchors in one feature map\n\n    Args:\n        X: shape=(4, 18, 25) the current anchor type for all points in the feature map\n        T: regression layer shape=(4, 18, 25)\n\n    Returns:\n        X: regressed position and size for current anchor\n    \"\"\"\n    try:\n        x = X[0, :, :]\n        y = X[1, :, :]\n        w = X[2, :, :]\n        h = X[3, :, :]\n\n        tx = T[0, :, :]\n        ty = T[1, :, :]\n        tw = T[2, :, :]\n        th = T[3, :, :]\n\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n\n        w1 = np.exp(tw.astype(np.float64)) * w\n        h1 = np.exp(th.astype(np.float64)) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n\n        x1 = np.round(x1)\n        y1 = np.round(y1)\n        w1 = np.round(w1)\n        h1 = np.round(h1)\n        return np.stack([x1, y1, w1, h1])\n    except Exception as e:\n        print(e)\n        return X\n    \ndef apply_regr(x, y, w, h, tx, ty, tw, th):\n    # Apply regression to x, y, w and h\n    try:\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n        w1 = math.exp(tw) * w\n        h1 = math.exp(th) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        w1 = int(round(w1))\n        h1 = int(round(h1))\n\n        return x1, y1, w1, h1\n\n    except ValueError:\n        return x, y, w, h\n    except OverflowError:\n        return x, y, w, h\n    except Exception as e:\n        print(e)\n        return x, y, w, h\n\ndef calc_iou(R, img_data, C, class_mapping):\n    \"\"\"Converts from (x1,y1,x2,y2) to (x,y,w,h) format\n\n    Args:\n        R: bboxes, probs\n    \"\"\"\n    bboxes = img_data['bboxes']\n    (width, height) = (img_data['width'], img_data['height'])\n    # get image dimensions for resizing\n    (resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n    gta = np.zeros((len(bboxes), 4))\n\n    for bbox_num, bbox in enumerate(bboxes):\n        # get the GT box coordinates, and resize to account for image resizing\n        # gta[bbox_num, 0] = (40 * (600 / 800)) / 16 = int(round(1.875)) = 2 (x in feature map)\n        gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))\n        gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))\n\n    x_roi = []\n    y_class_num = []\n    y_class_regr_coords = []\n    y_class_regr_label = []\n    IoUs = [] # for debugging only\n\n    # R.shape[0]: number of bboxes (=300 from non_max_suppression)\n    for ix in range(R.shape[0]):\n        (x1, y1, x2, y2) = R[ix, :]\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        x2 = int(round(x2))\n        y2 = int(round(y2))\n\n        best_iou = 0.0\n        best_bbox = -1\n        # Iterate through all the ground-truth bboxes to calculate the iou\n        for bbox_num in range(len(bboxes)):\n            curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])\n\n            # Find out the corresponding ground-truth bbox_num with larget iou\n            if curr_iou > best_iou:\n                best_iou = curr_iou\n                best_bbox = bbox_num\n\n        if best_iou < C.classifier_min_overlap:\n                continue\n        else:\n            w = x2 - x1\n            h = y2 - y1\n            x_roi.append([x1, y1, w, h])\n            IoUs.append(best_iou)\n\n            if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:\n                # hard negative example\n                cls_name = 'bg'\n            elif C.classifier_max_overlap <= best_iou:\n                cls_name = bboxes[best_bbox]['class']\n                cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0\n                cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0\n\n                cx = x1 + w / 2.0\n                cy = y1 + h / 2.0\n\n                tx = (cxg - cx) / float(w)\n                ty = (cyg - cy) / float(h)\n                tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))\n                th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))\n            else:\n                print('roi = {}'.format(best_iou))\n                raise RuntimeError\n\n        class_num = class_mapping[cls_name]\n        class_label = len(class_mapping) * [0]\n        class_label[class_num] = 1\n        y_class_num.append(copy.deepcopy(class_label))\n        coords = [0] * 4 * (len(class_mapping) - 1)\n        labels = [0] * 4 * (len(class_mapping) - 1)\n        if cls_name != 'bg':\n            label_pos = 4 * class_num\n            sx, sy, sw, sh = C.classifier_regr_std\n            coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]\n            labels[label_pos:4+label_pos] = [1, 1, 1, 1]\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n        else:\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n\n    if len(x_roi) == 0:\n        return None, None, None, None\n\n    # bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n    X = np.array(x_roi)\n    # one hot code for bboxes from above => x_roi (X)\n    Y1 = np.array(y_class_num)\n    # corresponding labels and corresponding gt bboxes\n    Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)\n\n    return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0), IoUs\ndef rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):\n\t\"\"\"Convert rpn layer to roi bboxes\n\n\tArgs: (num_anchors = 9)\n\t\trpn_layer: output layer for rpn classification \n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 18) if resized image is 400 width and 300\n\t\tregr_layer: output layer for rpn regression\n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 72) if resized image is 400 width and 300\n\t\tC: config\n\t\tuse_regr: Wether to use bboxes regression in rpn\n\t\tmax_boxes: max bboxes number for non-max-suppression (NMS)\n\t\toverlap_thresh: If iou in NMS is larger than this threshold, drop the box\n\n\tReturns:\n\t\tresult: boxes from non-max-suppression (shape=(300, 4))\n\t\t\tboxes: coordinates for bboxes (on the feature map)\n\t\"\"\"\n\tregr_layer = regr_layer / C.std_scaling\n\n\tanchor_sizes = C.anchor_box_scales   # (3 in here)\n\tanchor_ratios = C.anchor_box_ratios  # (3 in here)\n\n\tassert rpn_layer.shape[0] == 1\n\n\t(rows, cols) = rpn_layer.shape[1:3]\n\n\tcurr_layer = 0\n\n\t# A.shape = (4, feature_map.height, feature_map.width, num_anchors) \n\t# Might be (4, 18, 25, 18) if resized image is 400 width and 300\n\t# A is the coordinates for 9 anchors for every point in the feature map \n\t# => all 18x25x9=4050 anchors cooridnates\n\tA = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))\n\n\tfor anchor_size in anchor_sizes:\n\t\tfor anchor_ratio in anchor_ratios:\n\t\t\t# anchor_x = (128 * 1) / 16 = 8  => width of current anchor\n\t\t\t# anchor_y = (128 * 2) / 16 = 16 => height of current anchor\n\t\t\tanchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride\n\t\t\tanchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride\n\t\t\t\n\t\t\t# curr_layer: 0~8 (9 anchors)\n\t\t\t# the Kth anchor of all position in the feature map (9th in total)\n\t\t\tregr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4] # shape => (18, 25, 4)\n\t\t\tregr = np.transpose(regr, (2, 0, 1)) # shape => (4, 18, 25)\n\n\t\t\t# Create 18x25 mesh grid\n\t\t\t# For every point in x, there are all the y points and vice versa\n\t\t\t# X.shape = (18, 25)\n\t\t\t# Y.shape = (18, 25)\n\t\t\tX, Y = np.meshgrid(np.arange(cols),np. arange(rows))\n\n\t\t\t# Calculate anchor position and size for each feature map point\n\t\t\tA[0, :, :, curr_layer] = X - anchor_x/2 # Top left x coordinate\n\t\t\tA[1, :, :, curr_layer] = Y - anchor_y/2 # Top left y coordinate\n\t\t\tA[2, :, :, curr_layer] = anchor_x       # width of current anchor\n\t\t\tA[3, :, :, curr_layer] = anchor_y       # height of current anchor\n\n\t\t\t# Apply regression to x, y, w and h if there is rpn regression layer\n\t\t\tif use_regr:\n\t\t\t\tA[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)\n\n\t\t\t# Avoid width and height exceeding 1\n\t\t\tA[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])\n\n\t\t\t# Convert (x, y , w, h) to (x1, y1, x2, y2)\n\t\t\t# x1, y1 is top left coordinate\n\t\t\t# x2, y2 is bottom right coordinate\n\t\t\tA[2, :, :, curr_layer] += A[0, :, :, curr_layer]\n\t\t\tA[3, :, :, curr_layer] += A[1, :, :, curr_layer]\n\n\t\t\t# Avoid bboxes drawn outside the feature map\n\t\t\tA[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])\n\t\t\tA[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])\n\t\t\tA[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])\n\n\t\t\tcurr_layer += 1\n\n\tall_boxes = np.reshape(A.transpose((0, 3, 1, 2)), (4, -1)).transpose((1, 0))  # shape=(4050, 4)\n\tall_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))                   # shape=(4050,)\n\n\tx1 = all_boxes[:, 0]\n\ty1 = all_boxes[:, 1]\n\tx2 = all_boxes[:, 2]\n\ty2 = all_boxes[:, 3]\n\n\t# Find out the bboxes which is illegal and delete them from bboxes list\n\tidxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))\n\n\tall_boxes = np.delete(all_boxes, idxs, 0)\n\tall_probs = np.delete(all_probs, idxs, 0)\n\n\t# Apply non_max_suppression\n\t# Only extract the bboxes. Don't need rpn probs in the later process\n\tresult = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]\n\n\treturn result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = r'/kaggle/working/'\nos.chdir(base_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test=df_train[int(df_train.shape[0]*0.9):].copy()\ndf_train=df_train[:int(df_train.shape[0]*0.9)].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_preperation():\n    with open(\"train_wheat_annotation_pascal_voc.txt\", \"w+\") as file:\n        for idx, row in tqdm(df_train.iterrows(), desc = \"Files Processing Count\"):\n#             img=cv2.imread(row['image_id']) \n#             sys.stdout.write(str(idx) + '\\r')\n#             sys.stdout.flush()\n            file.write(row['image_id'] + ',' + str(int(row['x_min'])) + ',' + str(int(row['y_min'])) + ',' + str(int(row['x_max'])) + ',' + \n                       str(int(row['y_max'])) + ',' + row['source'] + '\\n')\n    with open(\"test_wheat_annotation_pascal_voc.txt\", \"w+\") as file:\n        for idx, row in tqdm(df_test.iterrows(), desc = \"Files Processing Count\"):\n#             img=cv2.imread(row['image_id'])\n#             sys.stdout.write(str(idx) + '\\r')\n#             sys.stdout.flush()\n            file.write(row['image_id'] + ',' + str(int(row['x_min'])) + ',' + str(int(row['y_min'])) + ',' + str(int(row['x_max'])) + ',' + \n                       str(int(row['y_max'])) + ',' + row['source'] + '\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_preperation()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.makedirs('model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path =  '/kaggle/working/train_wheat_annotation_pascal_voc.txt'\nnum_rois = 4\nhorizontal_flips = True # Augment with horizontal flips in training. \nvertical_flips = True   # Augment with vertical flips in training. \nrot_90 = True           # Augment with 90 degree rotations in training. \n\noutput_weight_path = os.path.join(base_path, 'model/model_frcnn_vgg.hdf5')\n\nrecord_path = os.path.join(base_path, 'model/record.csv')\nbase_weight_path = os.path.join(base_path, 'model/vgg16_weights_tf_dim_ordering_tf_kernels.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config_output_filename = os.path.join(base_path, 'model_vgg_config.pickle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the config\nC = Config()\nC.use_horizontal_flips = horizontal_flips\nC.use_vertical_flips = vertical_flips\nC.rot_90 = rot_90\nC.record_path = record_path\nC.model_path = output_weight_path\nC.num_rois = num_rois\nC.base_net_weights = base_weight_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C.base_net_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#--------------------------------------------------------#\n# This step will spend some time to load the data        #\n#--------------------------------------------------------#\nst = time.time()\ntrain_imgs, classes_count, class_mapping = get_data(train_path)\nprint()\nprint('Spend %0.2f mins to load the data' % ((time.time()-st)/60) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.savez_compressed('train_imgs.npz', train_imgs)\nnp.savez_compressed('classes_count.npz', classes_count)\nnp.savez_compressed('class_mapping.npz', class_mapping)\n# train_imgs=np.load('train_imgs.npz',allow_pickle=True)['arr_0'].tolist()\n# classes_count=np.load('classes_count.npz',allow_pickle=True)['arr_0'][()]\n# class_mapping=np.load('class_mapping.npz',allow_pickle=True)['arr_0'][()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pprint(train_imgs[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pprint(classes_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pprint(class_mapping)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 'bg' not in classes_count:\n\tclasses_count['bg'] = 0\n\tclass_mapping['bg'] = len(class_mapping)\n# e.g.\n#    classes_count: {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745, 'bg': 0}\n#    class_mapping: {'Person': 0, 'Car': 1, 'Mobile phone': 2, 'bg': 3}\nC.class_mapping = class_mapping\nprint('Training images per class:')\npprint(classes_count)\nprint('Num classes (including bg) = {}'.format(len(classes_count)))\nprint(class_mapping)\n# Save the configuration\nwith open(config_output_filename, 'wb') as config_f:\n\tpickle.dump(C,config_f)\n\tprint('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Shuffle the images with seed\nrandom.seed(1)\nrandom.shuffle(train_imgs)\nprint('Num train samples (images) {}'.format(len(train_imgs)))\ndata_gen_train = get_anchor_gt(train_imgs, C, get_img_output_length, mode='train')\nX, Y, image_data, debug_img, debug_num_pos = next(data_gen_train)\n\nprint(X.shape)\nprint(len(Y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Original image: height=%d width=%d'%(image_data['height'], image_data['width']))\nprint('Resized image:  height=%d width=%d C.im_size=%d'%(X.shape[1], X.shape[2], C.im_size))\nprint('Feature map size: height=%d width=%d C.rpn_stride=%d'%(Y[0].shape[1], Y[0].shape[2], C.rpn_stride))\nprint(X.shape)\nprint(str(len(Y))+\" includes 'y_rpn_cls' and 'y_rpn_regr'\")\nprint('Shape of y_rpn_cls {}'.format(Y[0].shape))\nprint('Shape of y_rpn_regr {}'.format(Y[1].shape))\nprint('Number of positive anchors for this image: %d' % (debug_num_pos))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if debug_num_pos==0:\n    gt_x1, gt_x2 = image_data['bboxes'][0]['x1']*(X.shape[2]/image_data['height']), image_data['bboxes'][0]['x2']*(X.shape[2]/image_data['height'])\n    gt_y1, gt_y2 = image_data['bboxes'][0]['y1']*(X.shape[1]/image_data['width']), image_data['bboxes'][0]['y2']*(X.shape[1]/image_data['width'])\n    gt_x1, gt_y1, gt_x2, gt_y2 = int(gt_x1), int(gt_y1), int(gt_x2), int(gt_y2)\n    img = debug_img.copy()\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    color = (0, 255, 0)\n    cv2.putText(img, 'gt bbox', (gt_x1, gt_y1-5), cv2.FONT_HERSHEY_DUPLEX, 0.7, color, 1)\n    cv2.rectangle(img, (gt_x1, gt_y1), (gt_x2, gt_y2), color, 2)\n    cv2.circle(img, (int((gt_x1+gt_x2)/2), int((gt_y1+gt_y2)/2)), 3, color, -1)\n    plt.grid()\n    plt.imshow(img)\n    plt.show()\nelse:\n    cls = Y[0][0]\n    pos_cls = np.where(cls==1)\n    regr = Y[1][0]\n    pos_regr = np.where(regr==1)\n    # print('y_rpn_cls for possible pos anchor: {}'.format(cls[pos_cls[0][0],pos_cls[1][0],:]))\n    # print('y_rpn_regr for positive anchor: {}'.format(regr[pos_regr[0][0],pos_regr[1][0],:]))\n    gt_x1, gt_x2 = image_data['bboxes'][0]['x1']*(X.shape[2]/image_data['width']), image_data['bboxes'][0]['x2']*(X.shape[2]/image_data['width'])\n    gt_y1, gt_y2 = image_data['bboxes'][0]['y1']*(X.shape[1]/image_data['height']), image_data['bboxes'][0]['y2']*(X.shape[1]/image_data['height'])\n    gt_x1, gt_y1, gt_x2, gt_y2 = int(gt_x1), int(gt_y1), int(gt_x2), int(gt_y2)\n    img = debug_img.copy()\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    color = (0, 255, 0)\n    #   cv2.putText(img, 'gt bbox', (gt_x1, gt_y1-5), cv2.FONT_HERSHEY_DUPLEX, 0.7, color, 1)\n    cv2.rectangle(img, (gt_x1, gt_y1), (gt_x2, gt_y2), color, 2)\n    cv2.circle(img, (int((gt_x1+gt_x2)/2), int((gt_y1+gt_y2)/2)), 3, color, -1)\n    # Add text\n    textLabel = 'gt bbox'\n    (retval,baseLine) = cv2.getTextSize(textLabel,cv2.FONT_HERSHEY_COMPLEX,0.5,1)\n    textOrg = (gt_x1, gt_y1+5)\n    cv2.rectangle(img, (textOrg[0] - 5, textOrg[1]+baseLine - 5), (textOrg[0]+retval[0] + 5, textOrg[1]-retval[1] - 5), (0, 0, 0), 2)\n    cv2.rectangle(img, (textOrg[0] - 5,textOrg[1]+baseLine - 5), (textOrg[0]+retval[0] + 5, textOrg[1]-retval[1] - 5), (255, 255, 255), -1)\n    cv2.putText(img, textLabel, textOrg, cv2.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 0), 1)\n    # Draw positive anchors according to the y_rpn_regr\n    for i in range(debug_num_pos):\n        color = (100+i*(155/4), 0, 100+i*(155/4))\n        idx = pos_regr[2][i*4]/4\n        anchor_size = C.anchor_box_scales[int(idx/3)]\n        anchor_ratio = C.anchor_box_ratios[2-int((idx+1)%3)]\n        center = (pos_regr[1][i*4]*C.rpn_stride, pos_regr[0][i*4]*C.rpn_stride)\n        # print('Center position of positive anchor: ', center)\n        cv2.circle(img, center, 3, color, -1)\n        anc_w, anc_h = anchor_size*anchor_ratio[0], anchor_size*anchor_ratio[1]\n        cv2.rectangle(img, (center[0]-int(anc_w/2), center[1]-int(anc_h/2)), (center[0]+int(anc_w/2), center[1]+int(anc_h/2)), color, 2)\n#         cv2.putText(img, 'pos anchor bbox '+str(i+1), (center[0]-int(anc_w/2), center[1]-int(anc_h/2)-5), cv2.FONT_HERSHEY_DUPLEX, 0.5, color, 1)\nprint('Green bboxes is ground-truth bbox. Others are positive anchors')\nplt.figure(figsize=(8,8))\nplt.grid()\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_shape_img = (None, None, 3)\nimg_input = Input(shape=input_shape_img)\nroi_input = Input(shape=(None, 4))\n# define the base network (VGG here, can be Resnet50, Inception, etc)\nshared_layers = nn_base(img_input, trainable=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget \"https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mv vgg16_weights_tf_dim_ordering_tf_kernels.h5 model/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !rm -r github.com","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) # 9\nrpn = rpn_layer(shared_layers, num_anchors)\nclassifier = classifier_layer(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count))\nmodel_rpn = Model(img_input, rpn[:2])\nmodel_classifier = Model([img_input, roi_input], classifier)\n# this is a model that holds both the RPN and the classifier, used to load/save weights for the models\nmodel_all = Model([img_input, roi_input], rpn[:2] + classifier)\n\n# Because the google colab can only run the session several hours one time (then you need to connect again), \n# we need to save the model and load the model to continue training\nif not os.path.isfile(C.model_path):\n    #If this is the begin of the training, load the pre-traind base network such as vgg-16\n    try:\n        print('This is the first time of your training')\n        print('loading weights from {}'.format(C.base_net_weights))\n        model_rpn.load_weights(C.base_net_weights, by_name=True)\n        model_classifier.load_weights(C.base_net_weights, by_name=True)\n    except:\n        print('Could not load pretrained model weights. Weights can be found in the keras application folder \\\n            https://github.com/fchollet/keras/tree/master/keras/applications')\n    \n    # Create the record.csv file to record losses, acc and mAP\n    record_df = pd.DataFrame(columns=['mean_overlapping_bboxes', 'class_acc', 'loss_rpn_cls', 'loss_rpn_regr', 'loss_class_cls', 'loss_class_regr', 'curr_loss', 'elapsed_time', 'mAP'])\nelse:\n    # If this is a continued training, load the trained model from before\n    print('Continue training based on previous trained model')\n    print('Loading weights from {}'.format(C.model_path))\n    model_rpn.load_weights(C.model_path, by_name=True)\n    model_classifier.load_weights(C.model_path, by_name=True)\n    \n    # Load the records\n    record_df = pd.read_csv(record_path)\n\n    r_mean_overlapping_bboxes = record_df['mean_overlapping_bboxes']\n    r_class_acc = record_df['class_acc']\n    r_loss_rpn_cls = record_df['loss_rpn_cls']\n    r_loss_rpn_regr = record_df['loss_rpn_regr']\n    r_loss_class_cls = record_df['loss_class_cls']\n    r_loss_class_regr = record_df['loss_class_regr']\n    r_curr_loss = record_df['curr_loss']\n    r_elapsed_time = record_df['elapsed_time']\n    r_mAP = record_df['mAP']\n\n    print('Already train %dK batches'% (len(record_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = Adam(lr=1e-5)\noptimizer_classifier = Adam(lr=1e-5)\nmodel_rpn.compile(optimizer=optimizer, loss=[rpn_loss_cls(num_anchors), rpn_loss_regr(num_anchors)])\nmodel_classifier.compile(optimizer=optimizer_classifier, loss=[class_loss_cls, class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})\nmodel_all.compile(optimizer='sgd', loss='mae')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training setting\ntotal_epochs = len(record_df)\nr_epochs = len(record_df)\nepoch_length = 1000\nnum_epochs = 20   #Just of sharing the kernel running with 2 epoch , you try with min 20 epochs\niter_num = 0\ntotal_epochs += num_epochs\nlosses = np.zeros((epoch_length, 5))\nrpn_accuracy_rpn_monitor = []\nrpn_accuracy_for_epoch = []\nif len(record_df)==0:\n    best_loss = np.Inf\nelse:\n    best_loss = np.min(r_curr_loss)\nprint(len(record_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nfor epoch_num in range(num_epochs):\n\n    progbar = generic_utils.Progbar(epoch_length)\n    print('Epoch {}/{}'.format(r_epochs + 1, total_epochs))\n    \n    r_epochs += 1\n\n    while True:\n        try:\n\n            if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:\n                mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor))/len(rpn_accuracy_rpn_monitor)\n                rpn_accuracy_rpn_monitor = []\n#                 print('Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'.format(mean_overlapping_bboxes, epoch_length))\n                if mean_overlapping_bboxes == 0:\n                    print('RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.')\n\n            # Generate X (x_img) and label Y ([y_rpn_cls, y_rpn_regr])\n            X, Y, img_data, debug_img, debug_num_pos = next(data_gen_train)\n\n            # Train rpn model and get loss value [_, loss_rpn_cls, loss_rpn_regr]\n            loss_rpn = model_rpn.train_on_batch(X, Y)\n\n            # Get predicted rpn from rpn model [rpn_cls, rpn_regr]\n            P_rpn = model_rpn.predict_on_batch(X)\n\n            # R: bboxes (shape=(300,4))\n            # Convert rpn layer to roi bboxes\n            R = rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_data_format(), use_regr=True, overlap_thresh=0.7, max_boxes=300)\n            \n            # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format\n            # X2: bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n            # Y1: one hot code for bboxes from above => x_roi (X)\n            # Y2: corresponding labels and corresponding gt bboxes\n            X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping)\n\n            # If X2 is None means there are no matching bboxes\n            if X2 is None:\n                rpn_accuracy_rpn_monitor.append(0)\n                rpn_accuracy_for_epoch.append(0)\n                continue\n            \n            # Find out the positive anchors and negative anchors\n            neg_samples = np.where(Y1[0, :, -1] == 1)\n            pos_samples = np.where(Y1[0, :, -1] == 0)\n\n            if len(neg_samples) > 0:\n                neg_samples = neg_samples[0]\n            else:\n                neg_samples = []\n\n            if len(pos_samples) > 0:\n                pos_samples = pos_samples[0]\n            else:\n                pos_samples = []\n\n            rpn_accuracy_rpn_monitor.append(len(pos_samples))\n            rpn_accuracy_for_epoch.append((len(pos_samples)))\n\n            if C.num_rois > 1:\n                # If number of positive anchors is larger than 4//2 = 2, randomly choose 2 pos samples\n                if len(pos_samples) < C.num_rois//2:\n                    selected_pos_samples = pos_samples.tolist()\n                else:\n                    selected_pos_samples = np.random.choice(pos_samples, C.num_rois//2, replace=False).tolist()\n                \n                # Randomly choose (num_rois - num_pos) neg samples\n                try:\n                    selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist()\n                except:\n                    selected_neg_samples = np.random.choice(neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist()\n                \n                # Save all the pos and neg samples in sel_samples\n                sel_samples = selected_pos_samples + selected_neg_samples\n            else:\n                # in the extreme case where num_rois = 1, we pick a random pos or neg sample\n                selected_pos_samples = pos_samples.tolist()\n                selected_neg_samples = neg_samples.tolist()\n                if np.random.randint(0, 2):\n                    sel_samples = random.choice(neg_samples)\n                else:\n                    sel_samples = random.choice(pos_samples)\n\n            # training_data: [X, X2[:, sel_samples, :]]\n            # labels: [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]\n            #  X                     => img_data resized image\n            #  X2[:, sel_samples, :] => num_rois (4 in here) bboxes which contains selected neg and pos\n            #  Y1[:, sel_samples, :] => one hot encode for num_rois bboxes which contains selected neg and pos\n            #  Y2[:, sel_samples, :] => labels and gt bboxes for num_rois bboxes which contains selected neg and pos\n            loss_class = model_classifier.train_on_batch([X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]])\n\n            losses[iter_num, 0] = loss_rpn[1]\n            losses[iter_num, 1] = loss_rpn[2]\n\n            losses[iter_num, 2] = loss_class[1]\n            losses[iter_num, 3] = loss_class[2]\n            losses[iter_num, 4] = loss_class[3]\n\n            iter_num += 1\n\n            progbar.update(iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])),\n                                      ('final_cls', np.mean(losses[:iter_num, 2])), ('final_regr', np.mean(losses[:iter_num, 3]))])\n\n            if iter_num == epoch_length:\n                loss_rpn_cls = np.mean(losses[:, 0])\n                loss_rpn_regr = np.mean(losses[:, 1])\n                loss_class_cls = np.mean(losses[:, 2])\n                loss_class_regr = np.mean(losses[:, 3])\n                class_acc = np.mean(losses[:, 4])\n\n                mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)\n                rpn_accuracy_for_epoch = []\n\n                if C.verbose:\n                    print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes))\n                    print('Classifier accuracy for bounding boxes from RPN: {}'.format(class_acc))\n                    print('Loss RPN classifier: {}'.format(loss_rpn_cls))\n                    print('Loss RPN regression: {}'.format(loss_rpn_regr))\n                    print('Loss Detector classifier: {}'.format(loss_class_cls))\n                    print('Loss Detector regression: {}'.format(loss_class_regr))\n                    print('Total loss: {}'.format(loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr))\n                    print('Elapsed time: {}'.format(time.time() - start_time))\n                    elapsed_time = (time.time()-start_time)/60\n\n                curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr\n                iter_num = 0\n                start_time = time.time()\n\n                if curr_loss < best_loss:\n                    if C.verbose:\n                        print('Total loss decreased from {} to {}, saving weights'.format(best_loss,curr_loss))\n                    best_loss = curr_loss\n                    model_all.save_weights(C.model_path)\n\n                new_row = {'mean_overlapping_bboxes':round(mean_overlapping_bboxes, 3), \n                           'class_acc':round(class_acc, 3), \n                           'loss_rpn_cls':round(loss_rpn_cls, 3), \n                           'loss_rpn_regr':round(loss_rpn_regr, 3), \n                           'loss_class_cls':round(loss_class_cls, 3), \n                           'loss_class_regr':round(loss_class_regr, 3), \n                           'curr_loss':round(curr_loss, 3), \n                           'elapsed_time':round(elapsed_time, 3), \n                           'mAP': 0}\n                record_df = record_df.append(new_row, ignore_index=True)\n                record_df.to_csv(record_path, index=0)\n                break\n        except Exception as e:\n            print('Exception: {}'.format(e))\n            continue\nprint('Training complete, exiting.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Inspection Graphs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['mean_overlapping_bboxes'], 'r')\nplt.title('mean_overlapping_bboxes')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['class_acc'], 'r')\nplt.title('class_acc')\n\nplt.show()\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['loss_rpn_cls'], 'r')\nplt.title('loss_rpn_cls')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['loss_rpn_regr'], 'r')\nplt.title('loss_rpn_regr')\nplt.show()\n\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['loss_class_cls'], 'r')\nplt.title('loss_class_cls')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['loss_class_regr'], 'r')\nplt.title('loss_class_regr')\nplt.show()\n\nplt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'r')\nplt.title('total_loss')\nplt.show()\n\n# plt.figure(figsize=(15,5))\n# plt.subplot(1,2,1)\n# plt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'r')\n# plt.title('total_loss')\n# plt.subplot(1,2,2)\n# plt.plot(np.arange(0, r_epochs), record_df['elapsed_time'], 'r')\n# plt.title('elapsed_time')\n# plt.show()\n\n# plt.title('loss')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_rpn_cls'], 'b')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_rpn_regr'], 'g')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_class_cls'], 'r')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_class_regr'], 'c')\n# # plt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'm')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['mean_overlapping_bboxes'], 'r')\nplt.title('mean_overlapping_bboxes')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['class_acc'], 'r')\nplt.title('class_acc')\n\nplt.show()\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['loss_rpn_cls'], 'r')\nplt.title('loss_rpn_cls')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['loss_rpn_regr'], 'r')\nplt.title('loss_rpn_regr')\nplt.show()\n\n\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.plot(np.arange(0, r_epochs), record_df['loss_class_cls'], 'r')\nplt.title('loss_class_cls')\nplt.subplot(1,2,2)\nplt.plot(np.arange(0, r_epochs), record_df['loss_class_regr'], 'r')\nplt.title('loss_class_regr')\nplt.show()\n\nplt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'r')\nplt.title('total_loss')\nplt.show()\n\n# plt.figure(figsize=(15,5))\n# plt.subplot(1,2,1)\n# plt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'r')\n# plt.title('total_loss')\n# plt.subplot(1,2,2)\n# plt.plot(np.arange(0, r_epochs), record_df['elapsed_time'], 'r')\n# plt.title('elapsed_time')\n# plt.show()\n\n# plt.title('loss')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_rpn_cls'], 'b')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_rpn_regr'], 'g')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_class_cls'], 'r')\n# plt.plot(np.arange(0, r_epochs), record_df['loss_class_regr'], 'c')\n# # plt.plot(np.arange(0, r_epochs), record_df['curr_loss'], 'm')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission  Code","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## All are JPGs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"assert len(glob.glob('/kaggle/input/global-wheat-detection/test/*')) == len([i for i in glob.glob('/kaggle/input/global-wheat-detection/test/*') if i.split('.')[1]=='jpg']) , \"All are not JPGs\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_img_list = np.array([ os.path.basename(i) for i in glob.glob('/kaggle/input/global-wheat-detection/test/*')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_load = \"/kaggle/input/model-wheat-detection/model_frcnn_vgg_111_epochs.hdf5\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def union(au, bu, area_intersection):\n\tarea_a = (au[2] - au[0]) * (au[3] - au[1])\n\tarea_b = (bu[2] - bu[0]) * (bu[3] - bu[1])\n\tarea_union = area_a + area_b - area_intersection\n\treturn area_union\n\n\ndef intersection(ai, bi):\n\tx = max(ai[0], bi[0])\n\ty = max(ai[1], bi[1])\n\tw = min(ai[2], bi[2]) - x\n\th = min(ai[3], bi[3]) - y\n\tif w < 0 or h < 0:\n\t\treturn 0\n\treturn w*h\n\n\ndef iou(a, b):\n\t# a and b should be (x1,y1,x2,y2)\n\n\tif a[0] >= a[2] or a[1] >= a[3] or b[0] >= b[2] or b[1] >= b[3]:\n\t\treturn 0.0\n\n\tarea_i = intersection(a, b)\n\tarea_u = union(a, b, area_i)\n\n\treturn float(area_i) / float(area_u + 1e-6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Couldn't complete from here. ..................................","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}