{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# set model and data directory\n# TODO: Add WBF https://www.kaggle.com/marcelosanchezortega/wbf-over-tta-single-model-efficientdet\n\nmodel_dir = '../input/wheat-mrcnn-weights/'\ndata_dir = '../input/global-wheat-detection'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#copy files to working directory + update model file so it works w/ tf2\n%cp -r ../input/mask-rcnn2/Mask_RCNN/Mask_RCNN ./\n%cp ../input/mask-rcnn2/model.py ./Mask_RCNN/mrcnn/model.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport ensemble_boxes\nimport imgaug\n\n%cd Mask_RCNN/\n\nfrom mrcnn.utils import Dataset, extract_bboxes\nfrom mrcnn.config import Config\nfrom mrcnn.model import MaskRCNN\nfrom mrcnn.visualize import display_instances\n%cd ../\n\n%ls\n\nimport os\nimport matplotlib\nimport ast\n\nimport keras\nprint(keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#setup mrcnn for the current system\n%cd Mask_RCNN/\n!python setup.py install\n%cd ../","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Functions for Wheat Images\nclass WheatDataset(Dataset):\n  # load the dataset definitions\n  def load_dataset(self, dataset_dir, is_train=True):\n    # define one class\n    self.add_class(\"dataset\", 1, \"wheat_head\")\n    # define data locations\n    images_dir = \"\"\n    aug_dir = []\n    if is_train:\n      images_dir = dataset_dir + '/train/'\n    else:\n      images_dir = dataset_dir + \"/test/\"\n\n    # find all images\n    for filename in os.listdir(images_dir):\n      # extract image id\n      image_id = filename[:-4]\n      # skip bad images\n      img_path = images_dir + filename\n      # add to dataset\n      self.add_image('dataset', image_id=image_id, path=img_path, annotation=img_path)\n\n  def extract_boxes(self, path):\n    return dataset_dicts[index_dict[path]][\"annotations\"], dataset_dicts[index_dict[path]][\"width\"], dataset_dicts[index_dict[path]][\"height\"]\n\n  # load the masks for an image\n  def load_mask(self, image_id):\n    # get details of image\n    info = self.image_info[image_id]\n    # define box file location\n    path = info['annotation']\n    # load XML\n    boxes, w, h = self.extract_boxes(path)\n    # create one array for all masks, each on a different channel\n    masks = np.zeros([h, w, len(boxes)], dtype='uint8')\n    # create masks\n    class_ids = list()\n    for i in range(len(boxes)):\n      box = boxes[i]\n      row_s, row_e = int(box[1]), int(box[3])\n      col_s, col_e = int(box[0]), int(box[2])\n      masks[row_s:row_e, col_s:col_e, i] = 1\n      class_ids.append(self.class_names.index('wheat_head'))\n    return masks, np.asarray(class_ids, dtype='int32')\n \t\n  # load an image reference\n  def image_reference(self, image_id):\n    info = self.image_info[image_id]\n    return info[\"path\"]\n\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test/val set\ntest_set = WheatDataset()\ntest_set.load_dataset(data_dir, is_train=False)\ntest_set.prepare()\nprint('Test: %d' % len(test_set.image_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the prediction configuration\nclass PredictionConfig(Config):\n\t# define the name of the configuration\n\tNAME = \"wheat_cfg\"\n\t# number of classes (background + wheat)\n\tNUM_CLASSES = 1 + 1\n\t# simplify GPU config\n\tGPU_COUNT = 1\n\tIMAGES_PER_GPU = 1\n\n\t#MAX_GT_INSTANCES = 200\n\n\tDETECTION_MIN_CONFIDENCE = 0.7\n\tDETECTION_NMS_THRESHOLD = 0.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create config\ncfg = PredictionConfig()\n# define the model\nmodel = MaskRCNN(mode='inference', model_dir=model_dir, config=cfg)\n# download the weights\nmodel.load_weights('../input/wheat-mrcnn-weights/mask_rcnn_wheat_cfg_0033.h5', by_name=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#WBF using notebook from Alex Shonenkov\nclass BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = 512\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = []\nfor image_id in dataset.image_ids:\n    image = dataset.load_image(image_id)\n\t\t# convert pixel values (e.g. center)\n    scaled_image = mold_image(image, cfg)\n\t\t# convert image into one sample\n    sample = expand_dims(scaled_image, 0)\n    images.append(sample)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use WBF\nfrom itertools import product\n\ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))\n\ndef make_tta_predictions(images, score_threshold=0.25):\n    predictions = []\n    for tta_transform in tta_transforms:\n        result = []\n        det = model(tta_transform.batch_augment(images.clone()))\n\n        for i in range(len(images)):\n            boxes = det[i][\"rois\"]    \n            scores = det[i][\"scores\"]\n            indexes = np.where(scores > score_threshold)[0]\n            boxes = boxes[indexes]\n            boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n            boxes = tta_transform.deaugment_boxes(boxes.copy())\n            result.append({\n                'boxes': boxes,\n                'scores': scores[indexes],\n            })\n        predictions.append(result)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot a number of photos with ground truth and predictions\nfrom os import listdir\nfrom xml.etree import ElementTree\nfrom numpy import zeros\nfrom numpy import asarray\nfrom numpy import expand_dims\nfrom matplotlib import pyplot\nfrom matplotlib.patches import Rectangle\nfrom mrcnn.config import Config\nfrom mrcnn.model import MaskRCNN\nfrom mrcnn.model import mold_image\nfrom mrcnn.utils import Dataset\n\n#Predict bboxes for images\nimport cv2\ndef predict_im(dir):\n  image = cv2.cvtColor(cv2.imread(dir), cv2.COLOR_BGR2RGB)\n  # convert pixel values (e.g. center)\n  scaled_image = mold_image(image, cfg)\n  # convert image into one sample\n  sample = expand_dims(scaled_image, 0)\n  # make prediction\n  yhat = model.detect(sample, verbose=0)[0]\n  # plot raw pixel data\n  pyplot.imshow(image)\n  ax = pyplot.gca()\n  \n  print(len(yhat['rois']))\n\n  # plot each box\n  for box in yhat['rois']:\n    # get coordinates\n    y1, x1, y2, x2 = box\n    # calculate width and height of the box\n    width, height = x2 - x1, y2 - y1\n    # create the shape\n    rect = Rectangle((x1, y1), width, height, fill=False, color='red')\n    # draw the box\n    ax.add_patch(rect)\n  pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for image_id in test_set.image_ids:\n  print(image_id)\n  predict_im(test_set.image_reference(image_id))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#compute the submission csv for the test images\n\ndef submission(dataset, model, cfg):\n  sub = pd.DataFrame(columns=[\"image_id\", \"PredictionString\"])\n\n  for image_id in dataset.image_ids:\n    image = dataset.load_image(image_id)\n\t\t# convert pixel values (e.g. center)\n    scaled_image = mold_image(image, cfg)\n\t\t# convert image into one sample\n    sample = expand_dims(scaled_image, 0)\n    yhat = model.detect(sample, verbose=0)\n\t\t# extract results for first sample\n    r = yhat[0]\n    predStr = \"\"\n\n    for (bbox, score) in zip(r[\"rois\"], r[\"scores\"]):\n      bbox[2] = bbox[2] - bbox[0]\n      bbox[3] = bbox[3] - bbox[1]\n\n      predStr = predStr + str(score) + \" \" + str(bbox[0]) + \" \" + str(bbox[1]) + \" \" + str(bbox[2]) + \" \" + str(bbox[3]) + \" \"\n\n    sub = sub.append({\"image_id\":dataset.image_reference(image_id)[-13:-4], \"PredictionString\":predStr}, ignore_index=True)\n\n  return sub\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import csv\n\nsub = submission(test_set, model, cfg)\nprint(sub)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final = True\n\nif final:\n    %rm -r Mask_RCNN/","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}