{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Car Image Segmentation Crop\n\n3차 대회는 이미지에서 차종을 예측하는 문제입니다.<br>\n제공된 데이터에 bounding box위치가 주어져 있으므로 이를 이용해 crop하여 사용할수 있지만,<br>\n박스안에는 다른 차나 배경, 사람등의 요소들이 포함되어 있기도 합니다.<br>\n따라서 박스안에 있는 차를 제외한 나머지 부분을 모두 잘라내면 성능이 높아질 것 같습니다. <br>\nMaskRCNN을 이용하여 segmentation/crop을 처리하는 커널을 작성해 보았습니다.<br>\n\n*참고자료<br>\nhttps://www.kaggle.com/fulrose/3rd-ml-month-car-model-classification-baseline <br>\nhttps://www.kaggle.com/tmheo74/3rd-ml-month-car-image-cropping-updated-7-10 <br>\nhttps://github.com/fizyr/keras-maskrcnn<br>"},{"metadata":{},"cell_type":"markdown","source":"# Common Task\n## Package Install/Load"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# package install\n!pip3 install keras-retinanet keras-maskrcnn\n\n# zip install\n!apt-get install zip\n\n# pretrained model download\n!mkdir model 2>/dev/null && cd model && wget https://github.com/fizyr/keras-maskrcnn/releases/download/0.2.2/resnet50_coco_v0.2.0.h5\n!ls -al model/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import keras\nimport keras\n\n# import keras_retinanet\nfrom keras_maskrcnn import models\nfrom keras_maskrcnn.utils.visualization import draw_mask\nfrom keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations\nfrom keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image\nfrom keras_retinanet.utils.colors import label_color\n\n# import miscellaneous modules\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport PIL\nfrom PIL import Image, ImageDraw\nfrom skimage import color as skcolor\n\n# set tf backend to allow memory to grow, instead of claiming everything\nimport tensorflow as tf\n\ndef get_session():\n    config = tf.ConfigProto()\n    config.gpu_options.allow_growth = True\n    return tf.Session(config=config)\n\n# use this environment flag to change which GPU to use\n#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n\n# set the modified tf session as backend in keras\nkeras.backend.tensorflow_backend.set_session(get_session())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MaskRcnn Model Load"},{"metadata":{"trusted":true},"cell_type":"code","source":"# MaskRcnn Model Load\nrcnn_model_path = './model/resnet50_coco_v0.2.0.h5'\nrcnn_model = models.load_model(rcnn_model_path, backbone_name='resnet50')\nrcnn_model.trainable = False\n#print(rcnn_model.summary())\n\n# load label to names mapping\nrcnn_labels_to_names = {0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus', 6: 'train', 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign', 12: 'parking meter', 13: 'bench', 14: 'bird', 15: 'cat', 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow', 20: 'elephant', 21: 'bear', 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag', 27: 'tie', 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', 32: 'sports ball', 33: 'kite', 34: 'baseball bat', 35: 'baseball glove', 36: 'skateboard', 37: 'surfboard', 38: 'tennis racket', 39: 'bottle', 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon', 45: 'bowl', 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', 51: 'carrot', 52: 'hot dog', 53: 'pizza', 54: 'donut', 55: 'cake', 56: 'chair', 57: 'couch', 58: 'potted plant', 59: 'bed', 60: 'dining table', 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote', 66: 'keyboard', 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink', 72: 'refrigerator', 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', 77: 'teddy bear', 78: 'hair drier', 79: 'toothbrush'}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Working Path & Data Check"},{"metadata":{"trusted":true},"cell_type":"code","source":"# DataSet 경로\nDATA_PATH = '../input/'\n\n# 이미지 폴더 경로\nTRAIN_IMG_PATH = os.path.join(DATA_PATH, 'train')\nTEST_IMG_PATH = os.path.join(DATA_PATH, 'test')\n\n# 작업 폴더\nTRAIN_CROP_PATH = \"./train_segcrop\"\nTEST_CROP_PATH = \"./test_segcrop\"\n\nfor dir in [TRAIN_CROP_PATH, TEST_CROP_PATH]:\n    if not os.path.exists(dir) :\n        os.mkdir(dir)\n    print(dir)\n\n# CSV 파일\ndf_train = pd.read_csv(os.path.join(DATA_PATH, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))\ndf_class = pd.read_csv(os.path.join(DATA_PATH, 'class.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CSV 파일 확인\ndisplay(df_train.describe())\ndisplay(df_test.describe())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Sample Image Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_index = 2999\ndf_bbox = df_train[sample_index:sample_index+1]\nprint(df_bbox)\nf_name = df_train['img_file'][sample_index]\n\nplt.figure(figsize=(9,12))\nplt.axis('off')\nimage = PIL.Image.open(os.path.join(TRAIN_IMG_PATH, f_name))\nplt.imshow(image)\nplt.show()\n\norg_bbox = df_bbox[['bbox_x1','bbox_y1','bbox_x2','bbox_y2']].values.reshape(-1)\nprint(org_bbox)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bbox_iou(boxA, boxB):\n    # determine the (x, y)-coordinates of the intersection rectangle\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n \n    # compute the area of intersection rectangle\n    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)\n \n    # compute the area of both the prediction and ground-truth\n    # rectangles\n    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)\n    boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)\n \n    # compute the intersection over union by taking the intersection\n    # area and dividing it by the sum of prediction + ground-truth\n    # areas - the interesection area\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n \n    # return the intersection over union value\n    return iou\n\nimport sys\ndef print2(string):\n    os.system(f'echo \\\"{string}\\\"')\n    print(string)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy to draw on\nimage = read_image_bgr(os.path.join(TRAIN_IMG_PATH, f_name))\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\ndraw_original = image.copy()\ndraw_singleboxed = image.copy()\ndraw_boxed = image.copy()\ndraw_shaped = image.copy()\n\ndraw_singlemask = image.copy()\ndraw_singlemask.fill(0)\n\n# preprocess image for network\nimage = preprocess_image(image)\nimage, scale = resize_image(image)\n\n# process image\nstart = time.time()\noutputs = rcnn_model.predict_on_batch(np.expand_dims(image, axis=0))\nprint(\"processing time: \", time.time() - start)\n\nboxes  = outputs[-4][0]\nscores = outputs[-3][0]\nlabels = outputs[-2][0]\nmasks  = outputs[-1][0]\n\n# correct for image scale\nboxes /= scale\n\nb0 = org_bbox.astype(int)\ndraw_box(draw_singleboxed, b0, color=[255,0,0])\nprint(\"org bbox = \", b0)\n\n# visualize detections\nmax_iou = -1\nmax_iou_box = None\nmax_iou_score = None\nmax_iou_label = None\nmax_iou_mask = None\n\nfor box, score, label, mask in zip(boxes, scores, labels, masks):\n    if score < 0.5:\n        break\n\n    color = label_color(label)\n    \n    b = box.astype(int)\n    print(\"bbox = \", b)\n    draw_box(draw_boxed, b, color=color)\n    \n    mask2 = mask[:, :, label]\n    draw_mask(draw_shaped, b, mask2, color=label_color(label))\n\n    tmp_iou = bbox_iou(b0, b)\n    if (tmp_iou > max_iou):\n        max_iou = tmp_iou\n        max_iou_box = b\n        max_iou_score = score\n        max_iou_label = label\n        max_iou_mask = mask2\n    \nprint(\"max_iou = \", max_iou)\ndraw_mask(draw_singlemask, max_iou_box, max_iou_mask, color=[255,255,255])\n   \n    \nfig, (ax1,ax2, ax3) = plt.subplots(1,3)\nfig.set_size_inches(18,12)\nax1.axis('off')\nax1.imshow(draw_original)\nax2.axis('off')\nax2.imshow(draw_boxed)\nax3.axis('off')\nax3.imshow(draw_shaped)\nplt.show()\n\n\nfig, (ax1,ax2,ax3) = plt.subplots(1,3)\nfig.set_size_inches(18,12)\nax1.axis('off')\nax1.imshow(draw_singleboxed)\n\ndraw_singlemask=np.where(draw_singlemask>1,255,0)\nax2.axis('off')\nax2.imshow(draw_singlemask)\n\nmasked_image = draw_original & draw_singlemask\narr_masked_image = Image.fromarray(masked_image.astype(np.uint8))\nfinal_image = arr_masked_image.crop(box=tuple(max_iou_box))\n\nax3.axis('off')\nax3.imshow(masked_image)\nplt.show()\n\nplt.imshow(final_image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Batch Image Processing "},{"metadata":{"trusted":true},"cell_type":"code","source":"for (in_path, out_path, df) in [(TRAIN_IMG_PATH, TRAIN_CROP_PATH, df_train), (TEST_IMG_PATH, TEST_CROP_PATH, df_test)]:\n  cnt = df['img_file'].count()\n  print(\"#path={} count={}\".format(out_path, cnt))\n  for idx in range(0, cnt):\n    #오래걸리므로 10개만 처리함(전체 처리시에는 아래부분 제거하시면 됩니다.)\n    if idx>=10:\n        break;\n        \n    #에러나는거 빼고는 다 돌아가야 한다.\n    try:\n\n        starttime = time.time()\n\n        df_row = df[idx:idx+1]\n        org_file_name = df_row['img_file'][idx]\n        #print(\"#{}\".format(org_file_name))\n        org_bbox = df_row[['bbox_x1','bbox_y1','bbox_x2','bbox_y2']].values.reshape(-1)\n        b0 = org_bbox.astype(int)\n\n        # copy to draw on\n        image = read_image_bgr(os.path.join(in_path, org_file_name))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        ctx_original = image.copy()\n        ctx_mask = image.copy()\n        ctx_mask.fill(0)\n\n        # preprocess image for network\n        image = preprocess_image(image)\n        image, scale = resize_image(image)\n\n        # process image\n        startpredict = time.time()\n        outputs = rcnn_model.predict_on_batch(np.expand_dims(image, axis=0))\n        #print(\"predict time: \", time.time() - startpredict)\n\n        boxes  = outputs[-4][0]\n        scores = outputs[-3][0]\n        labels = outputs[-2][0]\n        masks  = outputs[-1][0]\n\n        # correct for image scale\n        boxes /= scale\n\n        # lookup best detections\n        max_iou = -1\n        max_iou_box = None\n        max_iou_score = None\n        max_iou_label = None\n        max_iou_mask = None\n\n\n        for box, score, label, mask in zip(boxes, scores, labels, masks):\n            b = box.astype(int)\n            mask = mask[:, :, label]\n\n            # lookup max iou bbox\n            tmp_iou = bbox_iou(b0, b)\n            if (tmp_iou > max_iou):\n              max_iou = tmp_iou\n              max_iou_box = b\n              max_iou_score = score\n              max_iou_label = label\n              max_iou_mask = mask\n\n        # bbox overflow adjust\n        max_iou_box = np.where(max_iou_box<0, 0, max_iou_box)\n        if (max_iou_box[0] > ctx_mask.shape[1]) :\n            max_iou_box[0] = ctx_mask.shape[1]\n        if (max_iou_box[2] > ctx_mask.shape[1]) :\n            max_iou_box[2] = ctx_mask.shape[1]\n        if (max_iou_box[1] > ctx_mask.shape[0]) :\n            max_iou_box[1] = ctx_mask.shape[0]\n        if (max_iou_box[3] > ctx_mask.shape[0]) :\n            max_iou_box[3] = ctx_mask.shape[0]\n            \n        # create mask and crop \n        draw_mask(ctx_mask, max_iou_box, max_iou_mask, color=[255,255,255])    \n        ctx_mask=np.where(ctx_mask>1,255,0)\n\n        # create masked and crop \n        ctx_masked = ctx_original & ctx_mask\n        cropped_image = Image.fromarray(ctx_masked.astype(np.uint8)).crop(box=tuple(org_bbox))\n\n        cropped_image.save(os.path.join(out_path, org_file_name) )\n\n        print2(\"#{} => elapsed:{}s\".format(org_file_name, round(time.time() - starttime,2)))\n    except KeyboardInterrupt:\n        print(\"Stopped\")\n        sys.exit()\n    except:\n        print2(\"#{} => FAIL!\".format(org_file_name))\n        print2(sys.exc_info()[0])\n    #break;\n  #break;\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Result Check"},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_imgs = df_train['img_file'][0:5]\ntmp_imgs2 = df_test['img_file'][0:5]\nplt.figure(figsize=(16,16))\n\nfor num, f_name in enumerate(tmp_imgs):\n    img = PIL.Image.open(os.path.join(TRAIN_IMG_PATH, f_name))\n    plt.subplot(5, 4, 4*num + 1)\n    plt.title(f_name)\n    plt.imshow(img)\n    plt.axis('off')\n    \n    img_crop = PIL.Image.open(os.path.join(TRAIN_CROP_PATH, f_name))\n    plt.subplot(5, 4, 4*num + 2)\n    plt.title(f_name + ' cropped')\n    plt.imshow(img_crop)\n    plt.axis('off')\n    \nfor num, f_name in enumerate(tmp_imgs2):\n    img = PIL.Image.open(os.path.join(TEST_IMG_PATH, f_name))\n    plt.subplot(5, 4, 4*num + 3)\n    plt.title(f_name)\n    plt.imshow(img)\n    plt.axis('off')\n\n    img_crop = PIL.Image.open(os.path.join(TEST_CROP_PATH, f_name))\n    plt.subplot(5, 4, 4*num + 4)\n    plt.title(f_name + ' cropped')\n    plt.imshow(img_crop)\n    plt.axis('off')\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compress and remove files"},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf model \n!cd train_segcrop;zip ../train_segcrop.zip *.jpg > /dev/null\n!rm -rf train_segcrop\n!cd test_segcrop;zip ../test_segcrop.zip *.jpg > /dev/null\n!rm -rf test_segcrop","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}