{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Centernet Keras Inference\nIn this notebook I am going to make the inference sample for a simple centernet based model.\n\nThe training code can be found in [this notebook](https://www.kaggle.com/nvnnghia/keras-centernet-training)\n\nThe post processing parta are borrowed from [this notebook](https://www.kaggle.com/kmat2019/centernet-keypoint-detector)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2, os\nimport numpy as np \nfrom keras.utils import Sequence\nfrom imgaug import augmenters as iaa\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\n\nfrom PIL import Image\nfrom keras.layers import Dense,Dropout, Conv2D,Conv2DTranspose, BatchNormalization, Activation,AveragePooling2D,GlobalAveragePooling2D, Input, Concatenate, MaxPool2D, Add, UpSampling2D, LeakyReLU,ZeroPadding2D\nfrom keras.models import Model\n\nimport pandas as pd\n\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define the network","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"###\ncategory_n=1\noutput_layer_n=category_n+4\n\n##########MODEL#############\n\ndef aggregation_block(x_shallow, x_deep, deep_ch, out_ch):\n    x_deep= Conv2DTranspose(deep_ch, kernel_size=2, strides=2, padding='same', use_bias=False)(x_deep)\n    x_deep = BatchNormalization()(x_deep)   \n    x_deep = LeakyReLU(alpha=0.1)(x_deep)\n    x = Concatenate()([x_shallow, x_deep])\n    x=Conv2D(out_ch, kernel_size=1, strides=1, padding=\"same\")(x)\n    x = BatchNormalization()(x)   \n    x = LeakyReLU(alpha=0.1)(x)\n    return x\n  \n\n\ndef cbr(x, out_layer, kernel, stride):\n    x=Conv2D(out_layer, kernel_size=kernel, strides=stride, padding=\"same\")(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n    return x\n\ndef resblock(x_in,layer_n):\n    x=cbr(x_in,layer_n,3,1)\n    x=cbr(x,layer_n,3,1)\n    x=Add()([x,x_in])\n    return x  \n\n\n#I use the same network at CenterNet\ndef create_model(input_shape, aggregation=True):\n    input_layer = Input(input_shape)\n    \n    #resized input\n    input_layer_1=AveragePooling2D(2)(input_layer)\n    input_layer_2=AveragePooling2D(2)(input_layer_1)\n\n    #### ENCODER ####\n\n    x_0= cbr(input_layer, 16, 3, 2)#512->256\n    concat_1 = Concatenate()([x_0, input_layer_1])\n\n    x_1= cbr(concat_1, 32, 3, 2)#256->128\n    concat_2 = Concatenate()([x_1, input_layer_2])\n\n    x_2= cbr(concat_2, 64, 3, 2)#128->64\n    \n    x=cbr(x_2,64,3,1)\n    x=resblock(x,64)\n    x=resblock(x,64)\n    \n    x_3= cbr(x, 128, 3, 2)#64->32\n    x= cbr(x_3, 128, 3, 1)\n    x=resblock(x,128)\n    x=resblock(x,128)\n    x=resblock(x,128)\n    \n    x_4= cbr(x, 256, 3, 2)#32->16\n    x= cbr(x_4, 256, 3, 1)\n    x=resblock(x,256)\n    x=resblock(x,256)\n    x=resblock(x,256)\n    x=resblock(x,256)\n    x=resblock(x,256)\n \n    x_5= cbr(x, 512, 3, 2)#16->8\n    x= cbr(x_5, 512, 3, 1)\n    \n    x=resblock(x,512)\n    x=resblock(x,512)\n    x=resblock(x,512)\n    \n    #### DECODER ####\n    x_1= cbr(x_1, output_layer_n, 1, 1)\n    x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n    x_2= cbr(x_2, output_layer_n, 1, 1)\n    x_2 = aggregation_block(x_2, x_3, output_layer_n, output_layer_n)\n    x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n    x_3= cbr(x_3, output_layer_n, 1, 1)\n    x_3 = aggregation_block(x_3, x_4, output_layer_n, output_layer_n) \n    x_2 = aggregation_block(x_2, x_3, output_layer_n, output_layer_n)\n    x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n\n    x_4= cbr(x_4, output_layer_n, 1, 1)\n\n    x=cbr(x, output_layer_n, 1, 1)\n    x= UpSampling2D(size=(2, 2))(x)#8->16 \n\n    x = Concatenate()([x, x_4])\n    x=cbr(x, output_layer_n, 3, 1)\n    x= UpSampling2D(size=(2, 2))(x)#16->32\n\n    x = Concatenate()([x, x_3])\n    x=cbr(x, output_layer_n, 3, 1)\n    x= UpSampling2D(size=(2, 2))(x)#32->64 \n\n    x = Concatenate()([x, x_2])\n    x=cbr(x, output_layer_n, 3, 1)\n    x= UpSampling2D(size=(2, 2))(x)#64->128 \n\n    x = Concatenate()([x, x_1])\n    x=Conv2D(output_layer_n, kernel_size=3, strides=1, padding=\"same\")(x)\n    out = Activation(\"sigmoid\")(x)\n    \n    model=Model(input_layer, out)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Function for post processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def NMS_all(predicts,category_n, pred_out_h, pred_out_w, score_thresh,iou_thresh):\n    y_c=predicts[...,category_n]+np.arange(pred_out_h).reshape(-1,1)\n    x_c=predicts[...,category_n+1]+np.arange(pred_out_w).reshape(1,-1)\n    height=predicts[...,category_n+2]*pred_out_h\n    width=predicts[...,category_n+3]*pred_out_w\n\n    count=0\n    for category in range(category_n):\n        predict=predicts[...,category]\n        mask=(predict>score_thresh)\n        #print(\"box_num\",np.sum(mask))\n        if mask.all==False:\n            continue\n        box_and_score=NMS(predict[mask],y_c[mask],x_c[mask],height[mask],width[mask],iou_thresh,pred_out_h, pred_out_w)\n        box_and_score=np.insert(box_and_score,0,category,axis=1)#category,score,top,left,bottom,right\n        if count==0:\n            box_and_score_all=box_and_score\n        else:\n            box_and_score_all=np.concatenate((box_and_score_all,box_and_score),axis=0)\n        count+=1\n    score_sort=np.argsort(box_and_score_all[:,1])[::-1]\n    box_and_score_all=box_and_score_all[score_sort]\n    #print(box_and_score_all)\n\n    _,unique_idx=np.unique(box_and_score_all[:,2],return_index=True)\n    #print(unique_idx)\n    return box_and_score_all[sorted(unique_idx)]\n  \ndef NMS(score,y_c,x_c,height,width,iou_thresh,pred_out_h, pred_out_w,merge_mode=False):\n    if merge_mode:\n        score=score\n        top=y_c\n        left=x_c\n        bottom=height\n        right=width\n    else:\n        #flatten\n        score=score.reshape(-1)\n        y_c=y_c.reshape(-1)\n        x_c=x_c.reshape(-1)\n        height=height.reshape(-1)\n        width=width.reshape(-1)\n        size=height*width\n\n\n        top=y_c-height/2\n        left=x_c-width/2\n        bottom=y_c+height/2\n        right=x_c+width/2\n\n        inside_pic=(top>0)*(left>0)*(bottom<pred_out_h)*(right<pred_out_w)\n        outside_pic=len(inside_pic)-np.sum(inside_pic)\n        #if outside_pic>0:\n        #  print(\"{} boxes are out of picture\".format(outside_pic))\n        normal_size=(size<(np.mean(size)*20))*(size>(np.mean(size)/20))\n        score=score[inside_pic*normal_size]\n        top=top[inside_pic*normal_size]\n        left=left[inside_pic*normal_size]\n        bottom=bottom[inside_pic*normal_size]\n        right=right[inside_pic*normal_size]\n  \n\n    \n\n  #sort  \n    score_sort=np.argsort(score)[::-1]\n    score=score[score_sort]  \n    top=top[score_sort]\n    left=left[score_sort]\n    bottom=bottom[score_sort]\n    right=right[score_sort]\n\n    area=((bottom-top)*(right-left))\n\n    boxes=np.concatenate((score.reshape(-1,1),top.reshape(-1,1),left.reshape(-1,1),bottom.reshape(-1,1),right.reshape(-1,1)),axis=1)\n\n    box_idx=np.arange(len(top))\n    alive_box=[]\n    while len(box_idx)>0:\n  \n        alive_box.append(box_idx[0])\n\n        y1=np.maximum(top[0],top)\n        x1=np.maximum(left[0],left)\n        y2=np.minimum(bottom[0],bottom)\n        x2=np.minimum(right[0],right)\n\n        cross_h=np.maximum(0,y2-y1)\n        cross_w=np.maximum(0,x2-x1)\n        still_alive=(((cross_h*cross_w)/area[0])<iou_thresh)\n        if np.sum(still_alive)==len(box_idx):\n            print(\"error\")\n            print(np.max((cross_h*cross_w)),area[0])\n        top=top[still_alive]\n        left=left[still_alive]\n        bottom=bottom[still_alive]\n        right=right[still_alive]\n        area=area[still_alive]\n        box_idx=box_idx[still_alive]\n    return boxes[alive_box]#score,top,left,bottom,right\n\ndef visualize(box_and_score,img):\n    boxes = []\n    scores = []\n    colors= [(0,0,255), (255,0,0), (0,255,255), (0,127,127), (127,255,127), (255,255,0)]\n    classes = [\"car\", \"motor\", \"person\", \"bus\", \"truck\", \"bike\"]\n    number_of_rect=np.minimum(500,len(box_and_score))\n\n    for i in reversed(list(range(number_of_rect))):\n        predicted_class, score, top, left, bottom, right = box_and_score[i,:]\n\n\n        top = np.floor(top + 0.5).astype('int32')\n        left = np.floor(left + 0.5).astype('int32')\n        bottom = np.floor(bottom + 0.5).astype('int32')\n        right = np.floor(right + 0.5).astype('int32')\n\n        predicted_class = int(predicted_class)\n\n        label = '{:.2f}'.format(score)\n        #print(label)\n        #print(top, left, right, bottom)\n        cv2.rectangle(img, (left, top), (right, bottom), colors[predicted_class], 3)\n        cv2.putText(img, label, (left, top), cv2.FONT_HERSHEY_SIMPLEX ,  \n                       0.5, (255,255,255), 2, cv2.LINE_AA) \n        boxes.append([left, top, right-left, bottom-top])\n        scores.append(score)\n    \n    return np.array(boxes), np.array(scores)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Convert to submission format\nBorrowed from [here](https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR_INPUT = '/kaggle/input/global-wheat-detection'\nDIR_TRAIN = f'{DIR_INPUT}/train'\nDIR_TEST = f'{DIR_INPUT}/test'\n\nDIR_WEIGHTS = '/kaggle/input/kerascenter'\n\nWEIGHTS_FILE = f'{DIR_WEIGHTS}/atemp254-1.556.hdf5'\nprint(os.path.exists(WEIGHTS_FILE))\nimagenames = os.listdir(DIR_TEST)\n\n#def predict_image(imagenames,input_size=320, weights_file=''):\ninput_size = 768\n\npred_out_h=int(input_size/4)\npred_out_w=int(input_size/4)\n\nmodel=create_model(input_shape=(input_size,input_size,3))\n\nmodel.load_weights(WEIGHTS_FILE,by_name=True, skip_mismatch=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagenames","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\nfig, axes = plt.subplots(10, 1,figsize=(160,80))\nfor count, name in enumerate(imagenames):\n    ids = name.split('.')[0] \n    imagepath = '%s/%s.jpg'%(DIR_TEST,ids)\n    imgcv = cv2.imread(imagepath)\n    img = cv2.resize(imgcv, (input_size, input_size))\n    predict0 = model.predict((img[np.newaxis])/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n    #print(img.shape)\n    print_h, print_w = imgcv.shape[:2]\n    #print(predict.shape)\n    '''img90 =np.rot90(img)\n    predict90 = model.predict((img90[np.newaxis])/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n    predict90 = np.rot90(predict90, 3)\n    \n    img180 = np.rot90(img, 2)\n    predict180 = model.predict((img180[np.newaxis])/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n    predict180 = np.rot90(predict180, 2)\n    \n    img270 = np.rot90(img, 3)\n    predict270 = model.predict((img270[np.newaxis])/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n    predict270 = np.rot90(predict270)\n    \n    predict1 = np.add(predict0, predict90)/2\n    predict2 = np.add(predict180, predict270)/2\n    predict = np.add(predict1, predict2)/2'''\n    \n    box_and_score=NMS_all(predict0,category_n, pred_out_h, pred_out_w, score_thresh=0.25,iou_thresh=0.5)\n    if len(box_and_score)==0:\n        print('no boxes found!!')\n        #return\n        result = {\n                'image_id': ids,\n                'PredictionString': ''\n            }\n\n        results.append(result)\n    else:\n\n        #heatmap=predict[:,:,2]\n\n        box_and_score=box_and_score*[1,1,print_h/pred_out_h,print_w/pred_out_w,print_h/pred_out_h,print_w/pred_out_w]\n        # img=draw_rectangle(box_and_score[:,2:],img,\"red\")\n        # img=draw_rectangle(true_boxes,img,\"blue\")\n        preds, scores = visualize(box_and_score,imgcv)\n\n        result = {\n                'image_id': ids,\n                'PredictionString': format_prediction_string(preds, scores)\n            }\n\n        results.append(result)\n    \n\n    \n    # #axes[0].set_axis_off()\n    if count <10:\n        axes[count].imshow(imgcv)\n    # #axes[1].set_axis_off()\n    # axes[1].imshow(heatmap)#, cmap='gray')\n    # #axes[2].set_axis_off()\n    # #axes[2].imshow(heatmap_1)#, cmap='gray')\nplt.show()\n    #break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}