{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hello kagglers!\nI am a student at National Institute of Technology,Calicut in India. <br/>\nI am a deep learning and deep reinforcement learning enthusiast.<br/>\nThis kernel has scored 0.6032 on leaderboard. The main aim is to train a model which is fast and at the same time accurate.\n\n## Objectives:<br/>\n1. Speed along with accuracy<br/>\n2. Detecting small objects<br/>\n\n## Highlights of this kernel:\n\n1. Bi-fpn blocks in the architecture to ensure good fusion of higher and lower level feature vectors.\n2. Loss is penalized such that model learns smaller objects better.\n3. Both NMS and maxpooling can be used with this model.(NMS gave better results than maxpooling.)\n\n## Improvements that can be done or ideas that can be implemented:\n1. Splitting the data in a startified way. Considering, the height of bounding box to height of image or width of bounding box to width of image as parameter and splitting the data and ensembling the models might improve the performance.\n2. Mixup augmentation ( source : https://www.youtube.com/watch?v=a-VQfQqIMrE)\n3. Adding Noise\n\nYou can find the inference Kernel [here](https://www.kaggle.com/saimanojakondi/inference-kernel).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#installing efficientnet library\n!pip install efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#importing libraries\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.utils import Sequence\nfrom random import shuffle,choice,choices\nfrom tensorflow.keras.layers import *\nfrom sklearn.model_selection import StratifiedKFold\nfrom tensorflow.keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau,LearningRateScheduler\nfrom efficientnet import tfkeras as efn\nfrom tensorflow.keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train and test directories\ntrain_dir = '../input/global-wheat-detection/train'\ntrain_csv = '../input/global-wheat-detection/train.csv'\ntest_dir = '../input/global-wheat-detection/test'\n\n\nn_epochs = 40\nBATCH_SIZE = 2\nDISPLAY = 1\nlr = 0.001\nINPUT_SIZE = 512\nstride = 2\nOUTPUT_SIZE = INPUT_SIZE//4\nn_category = 1\noutput_layer_n = n_category + 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(train_csv)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_names = train_df['image_id'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#adding noise\ndef noisy(noise_typ,image):\n    if noise_typ == \"gauss\":\n        row,col,ch= image.shape\n        mean = 0\n        var = 0.\n        sigma = 1.0\n        gauss = np.random.normal(mean,sigma,(row,col,ch))\n        gauss = gauss.reshape(row,col,ch)\n        noisy = image/255 + gauss\n        return noisy*255\n    elif noise_typ == \"s&p\":\n        row,col,ch = image.shape\n        s_vs_p = 0.5\n        amount = 0.4\n        out = np.copy(image)\n      # Salt mode\n        num_salt = np.ceil(amount * image.size * s_vs_p)\n        coords = [np.random.randint(0, i - 1, int(num_salt))\n              for i in image.shape]\n        out[tuple(coords)] = 1\n\n      # Pepper mode\n        num_pepper = np.ceil(amount* image.size * (1. - s_vs_p))\n        coords = [np.random.randint(0, i - 1, int(num_pepper))\n              for i in image.shape]\n        out[tuple(coords)] = 0\n        return out\n    elif noise_typ == \"poisson\":\n        vals = len(np.unique(image))\n        vals = 2 ** np.ceil(np.log2(vals))\n        noisy = np.random.poisson(image * vals) / float(vals)\n        return noisy\n    elif noise_typ ==\"speckle\":\n        row,col,ch = image.shape\n        gauss = np.random.randn(row,col,ch)\n        gauss = gauss.reshape(row,col,ch)        \n        noisy = image + image * gauss\n        return noisy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#functions to extract the given data\ndef get_bb(string):\n    bbox = []\n    string = string[1:-1]\n    strings = string.split(',')\n    for char in strings:\n        bbox.append(int(float(char)))\n    return bbox\n#random visualization of bboxes\ndef random_visualize(train_dir,image_names,train_df,k):\n    k = k//2\n    fig,ax = plt.subplots(k,2,figsize = (10,10))\n    images = choices(image_names,k=k*2)\n\n    for i,image in enumerate(images):\n        \n        img = cv2.imread(train_dir+'/'+image+'.jpg')\n        ax[int(i//2)][int(i%2)].imshow(np.asarray(noisy('s&p',image=img)))\n        bboxes = train_df[train_df['image_id']==image]['bbox'].to_list()\n        for bbox in bboxes:\n            bbox = get_bb(bbox)\n            xmin = bbox[0]\n            ymin = bbox[1]\n            xmax = bbox[0]+bbox[2]\n            ymax = bbox[1]+bbox[3]\n            cv2.rectangle(img,(xmin,ymin),(xmax,ymax),(0,255,255),4)\n\n    plt.show()\nrandom_visualize(train_dir,image_names,train_df,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nwidths = []\nheights = []\nfor image in image_names:\n    image_dir = os.path.join(train_dir,image+'.jpg')\n    img = cv2.imread(image_dir)\n    im_w,im_h,c = img.shape[0],img.shape[1],img.shape[2]\n    bboxes = train_df[train_df['image_id']==image]['bbox'].to_list()\n    for bbox in bboxes:\n        bbox = get_bb(bbox)\n        widths.append(bbox[2]/im_w)\n        heights.append(bbox[3]/im_h)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nprint(plt.hist(widths,bins=10)[0])\nprint(plt.hist(widths,bins=10)[1])\nplt.show()\nplt.hist(heights,bins=10)\nplt.show()\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#generator for sequence\nclass My_generator(Sequence):\n    def __init__(self,image_names,train_df,train_dir=train_dir,input_size = INPUT_SIZE,batch_size = BATCH_SIZE,is_train = False,stride = stride):\n        self.image_names = image_names\n        self.train_df = train_df\n        self.stride = stride\n        self.train_dir = train_dir\n        self.batch_size = batch_size\n        self.input_size = input_size\n        self.output_size = self.input_size//self.stride\n        self.is_train = is_train\n        if self.is_train:\n            self.on_epoch_end()\n    def __len__(self):\n        return int(np.ceil(len(self.image_names)/float(self.batch_size)))\n    def on_epoch_end(self):\n        if self.is_train:\n            shuffle(image_names)\n    def __getitem__(self,idx):\n        batch_x = self.image_names[self.batch_size*idx:self.batch_size*(idx+1)]\n        if self.is_train:\n            return self.train_generator(batch_x)\n        else:\n            return self.valid_generator(batch_x)\n    def train_generator(self,batch_x):\n        X = []\n        heatmaps = []\n        Y = []\n        output_height,output_width = self.output_size,self.output_size\n        for image_name in list(batch_x):\n            image_dir = os.path.join(self.train_dir,image_name+'.jpg')\n            img = cv2.imread(image_dir)\n            im_w,im_h,c = img.shape[0],img.shape[1],img.shape[2]\n            img = cv2.resize(img,(self.input_size,self.input_size))\n            bboxes = train_df[train_df['image_id']==image_name]['bbox'].to_list()\n            output1 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n            output2 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n            output3 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n            output4 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n\n            for bbox in bboxes:\n                bbox = get_bb(bbox)\n                xc = bbox[0] + (bbox[2]/2)\n                yc = bbox[1] + (bbox[3]/2)\n                width = bbox[2]\n                height = bbox[3]\n                xc,yc,width,height = xc*(output_width/im_w),yc*(output_height/im_h),width*(output_width/im_w),height*(output_height/im_h)\n                category = 0\n                heatmap1=((np.exp(-(((np.arange(output_width)-xc)/(width/10))**2)/2)).reshape(1,-1)*(np.exp(-(((np.arange(output_height)-yc)/(height/10))**2)/2)).reshape(-1,1))\n                output1[:,:,0] = np.maximum(heatmap1[:,:],output1[:,:,0])\n                for i in range(n_category):\n                    output1[int(yc//1),int(xc//1),i+1] = 1\n                    output1[int(yc//1),int(xc//1),n_category+1] = yc%1\n                    output1[int(yc//1),int(xc//1),n_category+2] = xc%1\n                    output1[int(yc//1),int(xc//1),n_category+4] = width/output_width\n                    output1[int(yc//1),int(xc//1),n_category+3] = height/output_height\n                    \n                xc2 =  xc \n                yc2 = self.output_size-yc\n                heatmap2=((np.exp(-(((np.arange(output_width)-xc2)/(width/10))**2)/2)).reshape(1,-1)*(np.exp(-(((np.arange(output_height)-yc2)/(height/10))**2)/2)).reshape(-1,1))\n                output2[:,:,0] = np.maximum(heatmap2[:,:],output2[:,:,0])\n                for i in range(n_category):\n                    output2[int(yc2//1),int(xc2//1),i+1] = 1\n                    output2[int(yc2//1),int(xc2//1),n_category+1] = yc2%1\n                    output2[int(yc2//1),int(xc2//1),n_category+2] = xc2%1\n                    output2[int(yc2//1),int(xc2//1),n_category+4] = width/output_width\n                    output2[int(yc2//1),int(xc2//1),n_category+3] = height/output_height\n                    \n                xc3 = self.output_size - xc \n                yc3 = yc\n                heatmap3=((np.exp(-(((np.arange(output_width)-xc3)/(width/10))**2)/2)).reshape(1,-1)*(np.exp(-(((np.arange(output_height)-yc3)/(height/10))**2)/2)).reshape(-1,1))\n                output3[:,:,0] = np.maximum(output3[:,:,0],heatmap3[:,:])\n                for i in range(n_category):\n                    output3[int(yc3//1),int(xc3//1),i+1] = 1\n                    output3[int(yc3//1),int(xc3//1),n_category+1] = yc3%1\n                    output3[int(yc3//1),int(xc3//1),n_category+2] = xc3%1\n                    output3[int(yc3//1),int(xc3//1),n_category+4] = width/output_width\n                    output3[int(yc3//1),int(xc3//1),n_category+3] = height/output_height\n\n                xc4 = self.output_size-xc \n                yc4 = self.output_size-yc\n                heatmap4=((np.exp(-(((np.arange(output_width)-xc4)/(width/10))**2)/2)).reshape(1,-1)*(np.exp(-(((np.arange(output_height)-yc4)/(height/10))**2)/2)).reshape(-1,1))\n                output4[:,:,0] = np.maximum(output4[:,:,0],heatmap4[:,:])\n                for i in range(n_category):\n                    output4[int(yc4//1),int(xc4//1),i+1] = 1\n                    output4[int(yc4//1),int(xc4//1),n_category+1] = yc4%1\n                    output4[int(yc4//1),int(xc4//1),n_category+2] = xc4%1\n                    output4[int(yc4//1),int(xc4//1),n_category+4] = width/output_width\n                    output4[int(yc4//1),int(xc4//1),n_category+3] = height/output_height\n            image2 = cv2.flip(img,0)\n            image3 = cv2.flip(img,1)\n            image4 = cv2.flip(img,-1)\n            X.append(image2)\n            X.append(img)\n            X.append(image3)\n            X.append(image4)\n            Y.append(output2)\n            Y.append(output1)\n            Y.append(output3)\n            Y.append(output4)\n        X = np.asarray(X, np.float32)/255\n        Y = np.asarray(Y, np.float32)\n        return X,Y\n    def valid_generator(self,batch_x):\n        X = []\n        heatmaps = []\n        Y = []\n        output_height,output_width = self.output_size,self.output_size\n        for image_name in list(batch_x):\n            image_dir = os.path.join(self.train_dir,image_name+'.jpg')\n            img = cv2.imread(image_dir)\n            im_w,im_h,c = img.shape[0],img.shape[1],img.shape[2]\n            img = cv2.resize(img,(self.input_size,self.input_size))\n            bboxes = train_df[train_df['image_id']==image_name]['bbox'].to_list()\n            output1 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n            output2 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n            output3 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n            output4 = np.zeros((self.output_size,self.output_size,output_layer_n+n_category))\n\n            for bbox in bboxes:\n                bbox = get_bb(bbox)\n                xc = bbox[0] + (bbox[2]/2)\n                yc = bbox[1] + (bbox[3]/2)\n                width = bbox[2]\n                height = bbox[3]\n                xc,yc,width,height = xc*(output_width/im_w),yc*(output_height/im_h),width*(output_width/im_w),height*(output_height/im_h)\n                category = 0\n                heatmap1=((np.exp(-(((np.arange(output_width)-xc)/(width/10))**2)/2)).reshape(1,-1)*(np.exp(-(((np.arange(output_height)-yc)/(height/10))**2)/2)).reshape(-1,1))\n                output1[:,:,0] = np.maximum(heatmap1[:,:],output1[:,:,0])\n                for i in range(n_category):\n                    output1[int(yc//1),int(xc//1),i+1] = 1\n                    output1[int(yc//1),int(xc//1),n_category+1] = yc%1\n                    output1[int(yc//1),int(xc//1),n_category+2] = xc%1\n                    output1[int(yc//1),int(xc//1),n_category+4] = width/output_width\n                    output1[int(yc//1),int(xc//1),n_category+3] = height/output_height\n            X.append(img)\n            Y.append(output1)\n        X = np.asarray(X, np.float32)/255\n        Y = np.asarray(Y, np.float32)\n        #print(X.shape,Y.shape)\n        return X,Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing the generator\ndef test(i):\n    mygen = My_generator(image_names,train_df,train_dir=train_dir,batch_size=1, is_train = True)\n    #print(mygen)\n    X,Y = mygen.__getitem__(1)\n    X = X[i]\n    Y = Y[i]\n    X = np.asarray(np.ceil(X*255) ,np.uint8)\n    heatmap = Y[:,:,0]\n    points = np.argwhere(Y[:,:,1]==1)\n    #print(points)\n    for y,x in points:\n        offy = Y[y,x,2]\n        offx = Y[y,x,3]\n        width = Y[y,x,5]*(INPUT_SIZE/stride)\n        height = Y[y,x,4]*(INPUT_SIZE/stride)\n        xc = x+offx\n        yc = y+offy\n        xmin = int((xc-(width/2))*stride)\n        ymin = int((yc-(height/2))*stride)\n        xmax = int((xc+(width/2))*stride)\n        ymax = int((yc+(height/2))*stride)\n        cv2.rectangle(X, (xmin, ymin), (xmax, ymax), (0,255,255), 2)\n            #cv2.circle(X, (int(xc*4),int(yc*4)), 5, (0,0,255), 2) \n        #cv2.imshow('djpg',y[:,:,1]*255)\n        #cv2.imshow('drawjpg',x)\n    fig, ax = plt.subplots(1, 2, figsize=(16, 8))\n\n\n        #print(X.shape)\n    ax[0].imshow(X)\n    ax[1].imshow(heatmap)\n\n\ntest(0)\ntest(1)\ntest(2)\ntest(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fpn block and weighted-fpn block\ndef build_fpn(features,num_channels,wbifpn,kernel_size=2):\n    p4,p5,p6,p7 = features\n    #column1\n    p6 = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p6)\n    p5 = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p5)\n    p4 = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p4)\n    \n    p7 = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p7)\n    p7_resize = BatchNormalization()(p7)\n    p7_resize = MaxPool2D((kernel_size,kernel_size))(p7_resize)\n    if wbifpn:\n        p6_td = Fuse()([p6,p7_resize])\n    else:\n        p6_td = Add()([p6,p7_resize])\n    p6_td = Conv2D(num_channels,(3,3),kernel_initializer = 'glorot_uniform',activation='relu',padding='same')(p6_td)\n    p6_td = BatchNormalization()(p6_td)\n    p6_td = MaxPool2D((2,2),padding = 'same',strides = 1)(p6_td)\n    p6_td_resize = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p6_td)\n    p6_td_resize = BatchNormalization()(p6_td_resize)\n    \n    p6_td_resize = MaxPool2D((kernel_size,kernel_size))(p6_td_resize) \n    if wbifpn:\n        p5_td = Fuse()([p5,p6_td_resize])\n    else:\n        p5_td = Add()([p5,p6_td_resize])\n    p5_td = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p5_td)\n    p5_td = BatchNormalization()(p5_td)\n    p5_td = MaxPool2D((2,2),padding='same',strides = 1)(p5_td)\n    p5_td_resize = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p5_td)\n    p5_td_resize = BatchNormalization()(p5_td_resize)\n    p5_td_resize = MaxPooling2D((kernel_size,kernel_size))(p5_td_resize)\n    if wbifpn:\n        p4_td = Fuse()([p4,p5_td_resize])\n    else:\n        p4_td = Add()([p4,p5_td_resize])\n    p4_td = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p4_td)\n    p4_td = MaxPool2D((2,2),padding='same',strides = 1)(p4_td)\n    p4_U = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p4_td)\n    p4_U = BatchNormalization()(p4_U)\n    p5_U = UpSampling2D((kernel_size,kernel_size))(p4_U)\n    if wbifpn:\n        p5_U = Fuse()([p5,p5_td,p5_U])\n    else:\n        p5_U = Add()([p5,p5_td,p5_U])\n    p5_U = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p5_U)\n    p5_U = BatchNormalization()(p5_U)\n    p6_U = UpSampling2D((kernel_size,kernel_size))(p5_U)\n    if wbifpn:\n        p6_U = Fuse()([p6,p6_td,p6_U])\n    else:\n        p6_U = Add()([p6,p6_td,p6_U])\n    p6_U = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p6_U)\n    p6_U = BatchNormalization()(p6_U)\n    p7_U = UpSampling2D((kernel_size,kernel_size))(p6_U)\n    if wbifpn:\n        p7_U = Fuse()([p7,p7_U])\n    else:\n        p7_U = Add()([p7,p7_U])\n    p7_U = Conv2D(num_channels,(3,3),kernel_initializer='glorot_uniform',activation='relu',padding='same')(p7_U)\n    p7_U = BatchNormalization()(p7_U)\n    return (p4_U,p5_U,p6_U,p7_U)\n\n\nclass Fuse(tf.keras.layers.Layer):\n    '''Fusion layer'''\n    def __init__(self, epsilon=1e-4, **kwargs):\n        super(Fuse, self).__init__(**kwargs)\n        self.epsilon = epsilon\n\n    def build(self, input_shape):\n        num_in = len(input_shape)\n        self.w = self.add_weight(name=self.name,\n                                 shape=(num_in,),\n                                 initializer=tf.keras.initializers.constant(1 / num_in),\n                                 trainable=True,\n                                 dtype=tf.float32)\n\n    def call(self, inputs, **kwargs):\n        w = tf.keras.activations.relu(self.w)\n        x = tf.reduce_sum([w[i] * inputs[i] for i in range(len(inputs))], axis=0)\n        x = x / (tf.reduce_sum(w) + self.epsilon)\n        return x\n    def compute_output_shape(self, input_shape):\n        return input_shape[0]\n\n    def get_config(self):\n        config = super(Fuse, self).get_config()\n        config.update({\n            'epsilon': self.epsilon\n        })\n        return config\n    \ndef create_model(input_shape ,wbifpn=False):\n    '''model'''\n    effnet = efn.EfficientNetB4(input_shape=input_shape,weights=None,include_top = False)\n    p4 = effnet.get_layer('block2a_activation').output\n    p5 = effnet.get_layer('block3a_activation').output\n    p6 = effnet.get_layer('block4a_activation').output\n    p7 = effnet.get_layer('block7a_activation').output\n    features = (p7,p6,p5,p4)\n    features = build_fpn(features,16,wbifpn)\n    features = build_fpn(features,32,wbifpn)\n    features = build_fpn(features,64,wbifpn)\n    features = build_fpn(features,81,wbifpn)\n    features = list(features)\n\n    for i in range(1,4):\n        feature_curr = features[i]\n        feature_past = features[i-1]\n        feature_past_up = UpSampling2D((2,2))(feature_past)\n        feature_past_up = Conv2D(81,(3,3),padding='same',activation='relu',kernel_initializer='glorot_uniform')(feature_past_up)\n        if wbifpn:\n            feature_final = Fuse(name='final{}'.format(str(i)))([feature_curr,feature_past_up])\n        else:\n            feature_final = Add(name='final{}'.format(str(i)))([feature_curr,feature_past_up])\n        features[i] = feature_final\n    if stride == 2:\n        features[-1] = UpSampling2D((2,2))(features[-1])\n        features[-1] = Conv2D(128,(3,3),activation='relu',padding='same',kernel_initializer='glorot_uniform')(features[-1])\n    out = Conv2D(5,(3,3),activation='sigmoid',kernel_initializer='glorot_uniform',padding='same')(features[-1])\n    zeros = tf.expand_dims(tf.zeros_like(out[...,0]),axis=-1)\n    out_concat = tf.concat([zeros,out],axis = -1)\n    prediction_model=tf.keras.models.Model(inputs=[effnet.input],outputs=out)\n    model = Model(inputs = [effnet.input],outputs = out_concat)\n    return model,prediction_model\nmodel,prediction_model = create_model(input_shape=(512,512,3))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_loss(gamma,gamma2,y_true,y_pred,heatmaps):\n    '''focal loss'''\n    y_pred = K.clip(y_pred,1e-5,1-1e-5)\n    loglik = 2*y_true*((1-y_pred)**gamma)*K.log(y_pred) +(1-y_true)*((1-heatmaps)**gamma2)*(y_pred**gamma)*K.log(1-y_pred)\n    #heatloss=-K.sum(heatmap_true*((1-heatmap_pred)**alpha)*K.log(heatmap_pred+1e-6)+(1-heatmap_true)*((1-heatmap_true_rate)**beta)*(heatmap_pred**alpha)*K.log(1-heatmap_pred+1e-6))\n\n    cls_loss = -K.sum(loglik,axis=0)\n    return cls_loss\n\ndef loss_fn(gamma1,gamma2):\n    '''final combined loss and supporting functions'''\n    def final_loss(y_true,y_pred):\n        mask = K.batch_flatten(K.sign(y_true[...,4]))\n        N = K.sum(mask)\n        #heatmap loss\n        heatmaps = K.batch_flatten(y_true[...,0])\n        cls_pred = K.batch_flatten(y_pred[...,1])\n        cls_pred = K.clip(cls_pred,1e-7,1-1e-7)\n        cls_true = K.batch_flatten(y_true[...,1])\n\n        cls_loss = K.sum(focal_loss(gamma1,gamma2,cls_true,cls_pred,heatmaps))/N\n        \n        #offset loss\n        offy_pred = K.batch_flatten(y_pred[...,2])\n        offx_pred = K.batch_flatten(y_pred[...,3])\n        offy_true = K.batch_flatten(y_true[...,2])\n        offx_true = K.batch_flatten(y_true[...,3])\n        offloss = K.abs(offx_pred*mask-offx_true) + K.abs(offy_pred*mask-offy_true)\n        offloss = K.sum(offloss)/N\n        \n        #size loss\n        sizey_true = K.batch_flatten(y_true[...,4])\n        sizey_pred = K.batch_flatten(y_pred[...,4])\n        sizex_true = K.batch_flatten(y_true[...,5])\n        sizex_pred = K.batch_flatten(y_pred[...,5])\n        y_mask = tf.cast(sizey_pred>0.4,dtype=tf.float32)\n        x_mask = tf.cast(sizex_pred>0.4,dtype=tf.float32)\n        y_weight = y_mask*sizey_pred\n        x_weight = x_mask*sizex_pred\n        '''loss is penalized by 1+x_weight, 1+y_weight'''\n        size_loss = K.sum(K.abs(sizex_pred*mask-sizex_true)*(1+x_weight)+K.abs(sizey_pred*mask-sizey_true)*(1+y_weight))/N \n        loss = (1.5*cls_loss+1*offloss+10*size_loss)\n        return loss\n    return final_loss\ndef cls_metric(gamma1,gamma2):\n    def closs(y_true,y_pred):\n        mask = K.batch_flatten(K.sign(y_true[...,4]))\n        N = K.sum(mask)\n        heatmaps = K.batch_flatten(y_true[...,0])\n        cls_pred = K.batch_flatten(y_pred[...,1])\n        cls_pred = K.clip(cls_pred,1e-7,1-1e-7)\n        cls_true = K.batch_flatten(y_true[...,1])\n        cls_loss = K.sum(focal_loss(gamma1,gamma2,cls_true,cls_pred,heatmaps))/N\n        return cls_loss\n    return closs\n    \ndef off_loss(y_true,y_pred):\n    mask = K.batch_flatten(K.sign(y_true[...,4]))\n    N = K.sum(mask)\n    offy_pred = K.batch_flatten(y_pred[...,2])\n    offx_pred = K.batch_flatten(y_pred[...,3])\n    offy_true = K.batch_flatten(y_true[...,2])\n    offx_true = K.batch_flatten(y_true[...,3])\n    offloss = K.abs(offx_pred*mask-offx_true) + K.abs(offy_pred*mask-offy_true)\n    offloss = K.sum(offloss)/N\n    return offloss\ndef size_metric():\n    def sloss(y_true,y_pred):\n        mask = K.batch_flatten(K.sign(y_true[...,4]))\n        N = K.sum(mask)\n        sizey_true = K.batch_flatten(y_true[...,4])\n        sizey_pred = K.batch_flatten(y_pred[...,4])\n        sizex_true = K.batch_flatten(y_true[...,5])\n        sizex_pred = K.batch_flatten(y_pred[...,5])\n        y_mask = tf.cast(sizey_pred>0.4,dtype=tf.float32)\n        x_mask = tf.cast(sizex_pred>0.4,dtype=tf.float32)\n        y_weight = y_mask*sizey_pred\n        x_weight = x_mask*sizex_pred\n        size_loss = K.sum(K.abs(sizex_pred*mask-sizex_true)*(1+x_weight)+K.abs(sizey_pred*mask-sizey_true)*(1+y_weight))/N#size_loss2 = K.sum(K.abs(sizex_pred*mask-sizex_true)*(1-sizex_true)*mask+K.abs(sizey_pred*mask-sizey_true)*(1-sizey_true)*mask,axis=-1)\n        #size_loss = 0.8*size_loss1+0.2*size_loss2\n        return size_loss\n    return sloss\ndef build_model(input_shape,gamma1=1.5,gamma2=3.0,lr=lr):\n    model,_ = create_model(input_shape)\n    optimizer = Adam(lr = lr)\n    model.compile(loss = loss_fn(gamma1,gamma2),optimizer = optimizer,metrics = [off_loss,size_metric(),cls_metric(gamma1,gamma2)])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learning rate scheduler\ndef lrs(epoch):\n    lr = 0.001\n    if epoch >= 20: lr = 0.0002\n    return lr\n\nlr_schedule = LearningRateScheduler(lrs)\nearly_stopping = EarlyStopping(monitor = 'val_loss', min_delta=0, patience = 5, verbose = 1)\nprint('no of available datapoints : {}'.format(4*len(image_names)))\n#splitting\ncurT = image_names[300:]\ncurV = image_names[:300]\n#generators\ntrain_gen = My_generator(curT,train_df,batch_size = BATCH_SIZE,is_train=True)\nval_gen = My_generator(curV,train_df,batch_size = BATCH_SIZE,is_train = False)\nmodel = build_model((INPUT_SIZE,INPUT_SIZE,3))\nSTEPS_PER_EPOCH = curT.shape[0]//(BATCH_SIZE)\nname = 'trained.h5'\ncheckpoint = ModelCheckpoint(name,monitor = 'val_loss', save_best_only = True, verbose = 1, period = 1)\n#training\nhistory = model.fit_generator(train_gen,epochs = n_epochs ,verbose = DISPLAY,steps_per_epoch = STEPS_PER_EPOCH,validation_data = val_gen,shuffle = True,validation_steps = curV.shape[0]//(BATCH_SIZE),callbacks = [lr_schedule,checkpoint,early_stopping])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}