{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Mask R-CNN - Inspect Nucleus Trained Model\n\nCode and visualizations to test, debug, and evaluate the Mask R-CNN model."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install mrcnn\nfrom shutil import copyfile\ncopyfile(src = \"../input/nucleus/nucleus.py\", dst = \"../working/nucleus.py\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport random\nimport math\nimport re\nimport time\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\n# Root directory of the project\nROOT_DIR = os.path.abspath(\"../../\")\n\n# Import Mask RCNN\nsys.path.append(ROOT_DIR)  # To find local version of the library\nfrom mrcnn import utils\nfrom mrcnn import visualize\nfrom mrcnn.visualize import display_images\nimport mrcnn.model as modellib\nfrom mrcnn.model import log\n\nimport nucleus\n\n%matplotlib inline \n\n# Directory to save logs and trained model\nLOGS_DIR = os.path.join(ROOT_DIR, \"kaggle/input\")\nCOCO_MODEL_PATH = os.path.join(LOGS_DIR, \"coco-weights/mask_rcnn_coco_v1.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../../kaggle/input/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Comment out to reload imported modules if they change\n# %load_ext autoreload\n# %autoreload 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configurations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset directory\nDATASET_DIR = os.path.join(ROOT_DIR, \"kaggle/input/data-science-bowl-2018\")\n\n# Inference Configuration\nconfig = nucleus.NucleusInferenceConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Notebook Preferences"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Device to load the neural network on.\n# Useful if you're training a model on the same \n# machine, in which case use CPU and leave the\n# GPU for training.\nDEVICE = \"/gpu:0\"  # /cpu:0 or /gpu:0\n\n# Inspect the model in training or inference modes\n# values: 'inference' or 'training'\n# Only inference mode is supported right now","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ax(rows=1, cols=1, size=16):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    fig, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    fig.tight_layout()\n    return ax","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load training dataset\ndataset_train = nucleus.NucleusDataset()\ndataset_train.load_nucleus(DATASET_DIR, \"train\")\ndataset_train.prepare()\n\nprint(\"Images: {}\\nClasses: {}\".format(len(dataset_train.image_ids), dataset_train.class_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load validation dataset\ndataset_val = nucleus.NucleusDataset()\ndataset_val.load_nucleus(DATASET_DIR, \"val\")\ndataset_val.prepare()\n\nprint(\"Images: {}\\nClasses: {}\".format(len(dataset_val.image_ids), dataset_val.class_names))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Model"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"# Create model in inference mode\nwith tf.device(DEVICE):\n    model = modellib.MaskRCNN(mode=\"training\", config=config,\n                          model_dir=LOGS_DIR)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"# Which weights to start with?\n\ninit_with = \"coco\"  # imagenet, coco, or last\n\nif init_with == \"imagenet\":\n    model.load_weights(model.get_imagenet_weights(), by_name=True)\nelif init_with == \"coco\":\n    # Load weights trained on MS COCO, but skip layers that\n    # are different due to the different number of classes\n    # See README for instructions to download the COCO weights\n    model.load_weights(COCO_MODEL_PATH, by_name=True,\n                       exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \n                                \"mrcnn_bbox\", \"mrcnn_mask\"])\nelif init_with == \"last\":\n    # Load the last model you trained and continue training\n    Last_model_path = \"../input/nucleus20190417T0142/mask_rcnn_nucleus_0002.h5\"\n    model.load_weights(Last_model_path, by_name=True, exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \n                                \"mrcnn_bbox\", \"mrcnn_mask\"])\n\n\n\n#model.load_weights(model.get_imagenet_weights(), by_name=True,exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \n#                                \"mrcnn_bbox\", \"mrcnn_mask\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.train(dataset_train, dataset_val, \n            learning_rate=config.LEARNING_RATE, \n            epochs=4, \n            layers='heads')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.train(dataset_train, dataset_val, \n            learning_rate=config.LEARNING_RATE / 10,\n            epochs=5, \n            layers=\"all\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run Detection"},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(\"../input/nucleus20190417T0456\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = nucleus.NucleusInferenceConfig()\nconfig.display()\n\n# Recreate the model in inference mode\nmodel_detect = modellib.MaskRCNN(mode=\"inference\", \n                          config=config,\n                          model_dir=LOGS_DIR)\n\n# Get path to saved weights\n# Either set a specific path or find last trained weights\n# model_path = os.path.join(ROOT_DIR, \".h5 file name here\")\nLast_model_path = \"../input/nucleus20190417T0456/mask_rcnn_nucleus_0005.h5\"\nmodel_path = Last_model_path\n\n# Load trained weights\nprint(\"Loading weights from \", model_path)\nmodel_detect.load_weights(model_path, by_name=True, exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \n                                \"mrcnn_bbox\", \"mrcnn_mask\"])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"image_id = random.choice(dataset_val.image_ids)\nimage, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n    modellib.load_image_gt(dataset_val, config, image_id, use_mini_mask=False)\ninfo = dataset_val.image_info[image_id]\nprint(\"image ID: {}.{} ({}) {}\".format(info[\"source\"], info[\"id\"], image_id, \n                                       dataset_val.image_reference(image_id)))\nprint(\"Original image shape: \", modellib.parse_image_meta(image_meta[np.newaxis,...])[\"original_image_shape\"][0])\n\n# Run object detection\nresults = model_detect.detect_molded(np.expand_dims(image, 0), np.expand_dims(image_meta, 0), verbose=1)\n\n# Display results\nr = results[0]\nlog(\"gt_class_id\", gt_class_id)\nlog(\"gt_bbox\", gt_bbox)\nlog(\"gt_mask\", gt_mask)\n\n# Compute AP over range 0.5 to 0.95 and print it\nutils.compute_ap_range(gt_bbox, gt_class_id, gt_mask,\n                       r['rois'], r['class_ids'], r['scores'], r['masks'],\n                       verbose=1)\n\nvisualize.display_differences(\n    image,\n    gt_bbox, gt_class_id, gt_mask,\n    r['rois'], r['class_ids'], r['scores'], r['masks'],\n    dataset_val.class_names, ax=get_ax(),\n    show_box=False, show_mask=False,\n    iou_threshold=0.5, score_threshold=0.5)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"# Display predictions only\nvisualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], \n                            dataset_val.class_names, r['scores'], ax=get_ax(1),\n                            show_bbox=False, show_mask=False,\n                            title=\"Predictions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display Ground Truth only\n# visualize.display_instances(image, gt_bbox, gt_mask, gt_class_id, \n#                             dataset.class_names, ax=get_ax(1),\n#                             show_bbox=False, show_mask=False,\n#                             title=\"Ground Truth\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Compute AP on Batch of Images"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"def compute_batch_ap(dataset, image_ids, verbose=1):\n    APs = []\n    for image_id in image_ids:\n        # Load image\n        image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n            modellib.load_image_gt(dataset, config,\n                                   image_id, use_mini_mask=False)\n        print(\"IMAGE SHAPE: \", image.shape)\n        # Run object detection\n        results = model_detect.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0)\n        # Compute AP over range 0.5 to 0.95\n        r = results[0]\n        ap = utils.compute_ap_range(\n            gt_bbox, gt_class_id, gt_mask,\n            r['rois'], r['class_ids'], r['scores'], r['masks'],\n            verbose=1)\n        APs.append(ap)\n        if verbose:\n            info = dataset.image_info[image_id]\n            meta = modellib.parse_image_meta(image_meta[np.newaxis,...])\n            print(\"{:3} {}   AP: {:.2f}\".format(\n                meta[\"image_id\"][0], meta[\"original_image_shape\"][0], ap))\n    return APs\n\n# Run on validation set\nlimit = 3\nAPs = compute_batch_ap(dataset_val, dataset_val.image_ids[:limit])\nprint(\"Mean AP overa {} images: {:.4f}\".format(len(APs), np.mean(APs)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Step by Step Prediction"},{"metadata":{},"cell_type":"markdown","source":"## Stage 1: Region Proposal Network\n\nThe Region Proposal Network (RPN) runs a lightweight binary classifier on a lot of boxes (anchors) over the image and returns object/no-object scores. Anchors with high *objectness* score (positive anchors) are passed to the stage two to be classified.\n\nOften, even positive anchors don't cover objects fully. So the RPN also regresses a refinement (a delta in location and size) to be applied to the anchors to shift it and resize it a bit to the correct boundaries of the object."},{"metadata":{},"cell_type":"markdown","source":"### 1.a RPN Targets\n\nThe RPN targets are the training values for the RPN. To generate the targets, we start with a grid of anchors that cover the full image at different scales, and then we compute the IoU of the anchors with ground truth object. Positive anchors are those that have an IoU >= 0.7 with any ground truth object, and negative anchors are those that don't cover any object by more than 0.3 IoU. Anchors in between (i.e. cover an object by IoU >= 0.3 but < 0.7) are considered neutral and excluded from training.\n\nTo train the RPN regressor, we also compute the shift and resizing needed to make the anchor cover the ground truth object completely."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get anchors and convert to pixel coordinates\nanchors = model_detect.get_anchors(image.shape)\nanchors = utils.denorm_boxes(anchors, image.shape[:2])\nlog(\"anchors\", anchors)\n\n# Generate RPN trainig targets\n# target_rpn_match is 1 for positive anchors, -1 for negative anchors\n# and 0 for neutral anchors.\ntarget_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(\n    image.shape, anchors, gt_class_id, gt_bbox, model.config)\nlog(\"target_rpn_match\", target_rpn_match)\nlog(\"target_rpn_bbox\", target_rpn_bbox)\n\npositive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]\nnegative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]\nneutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]\npositive_anchors = anchors[positive_anchor_ix]\nnegative_anchors = anchors[negative_anchor_ix]\nneutral_anchors = anchors[neutral_anchor_ix]\nlog(\"positive_anchors\", positive_anchors)\nlog(\"negative_anchors\", negative_anchors)\nlog(\"neutral anchors\", neutral_anchors)\n\n# Apply refinement deltas to positive anchors\nrefined_anchors = utils.apply_box_deltas(\n    positive_anchors,\n    target_rpn_bbox[:positive_anchors.shape[0]] * model_detect.config.RPN_BBOX_STD_DEV)\nlog(\"refined_anchors\", refined_anchors, )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display positive anchors before refinement (dotted) and\n# after refinement (solid).\nvisualize.draw_boxes(\n    image, ax=get_ax(),\n    boxes=positive_anchors,\n    refined_boxes=refined_anchors)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.b RPN Predictions\n\nHere we run the RPN graph and display its predictions."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run RPN sub-graph\npillar = model_detect.keras_model.get_layer(\"ROI\").output  # node to start searching from\n\n# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10\nnms_node = model_detect.ancestor(pillar, \"ROI/rpn_non_max_suppression:0\")\nif nms_node is None:\n    nms_node = model_detect.ancestor(pillar, \"ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0\")\nif nms_node is None: #TF 1.9-1.10\n    nms_node = model_detect.ancestor(pillar, \"ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0\")\n\nrpn = model_detect.run_graph(image[np.newaxis], [\n    (\"rpn_class\", model_detect.keras_model.get_layer(\"rpn_class\").output),\n    (\"pre_nms_anchors\", model_detect.ancestor(pillar, \"ROI/pre_nms_anchors:0\")),\n    (\"refined_anchors\", model_detect.ancestor(pillar, \"ROI/refined_anchors:0\")),\n    (\"refined_anchors_clipped\", model_detect.ancestor(pillar, \"ROI/refined_anchors_clipped:0\")),\n    (\"post_nms_anchor_ix\", nms_node),\n    (\"proposals\", model_detect.keras_model.get_layer(\"ROI\").output),\n], image_metas=image_meta[np.newaxis])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show top anchors by score (before refinement)\nlimit = 100\nsorted_anchor_ids = np.argsort(rpn['rpn_class'][:,:,1].flatten())[::-1]\nvisualize.draw_boxes(image, boxes=anchors[sorted_anchor_ids[:limit]], ax=get_ax())","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"# Show top anchors with refinement. Then with clipping to image boundaries\nlimit = 50\nax = get_ax(1, 2)\npre_nms_anchors = utils.denorm_boxes(rpn[\"pre_nms_anchors\"][0], image.shape[:2])\nrefined_anchors = utils.denorm_boxes(rpn[\"refined_anchors\"][0], image.shape[:2])\nrefined_anchors_clipped = utils.denorm_boxes(rpn[\"refined_anchors_clipped\"][0], image.shape[:2])\nvisualize.draw_boxes(image, boxes=pre_nms_anchors[:limit],\n                     refined_boxes=refined_anchors[:limit], ax=ax[0])\nvisualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[:limit], ax=ax[1])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"# Show refined anchors after non-max suppression\nlimit = 50\nixs = rpn[\"post_nms_anchor_ix\"][:limit]\nvisualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[ixs], ax=get_ax())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show final proposals\n# These are the same as the previous step (refined anchors \n# after NMS) but with coordinates normalized to [0, 1] range.\nlimit = 50\n# Convert back to image coordinates for display\n# h, w = config.IMAGE_SHAPE[:2]\n# proposals = rpn['proposals'][0, :limit] * np.array([h, w, h, w])\nvisualize.draw_boxes(\n    image, ax=get_ax(),\n    refined_boxes=utils.denorm_boxes(rpn['proposals'][0, :limit], image.shape[:2]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stage 2: Proposal Classification\n\nThis stage takes the region proposals from the RPN and classifies them."},{"metadata":{},"cell_type":"markdown","source":"### 2.a Proposal Classification\n\nRun the classifier heads on proposals to generate class propbabilities and bounding box regressions."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get input and output to classifier and mask heads.\nmrcnn = model_detect.run_graph([image], [\n    (\"proposals\", model_detect.keras_model.get_layer(\"ROI\").output),\n    (\"probs\", model_detect.keras_model.get_layer(\"mrcnn_class\").output),\n    (\"deltas\", model_detect.keras_model.get_layer(\"mrcnn_bbox\").output),\n    (\"masks\", model_detect.keras_model.get_layer(\"mrcnn_mask\").output),\n    (\"detections\", model_detect.keras_model.get_layer(\"mrcnn_detection\").output),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get detection class IDs. Trim zero padding.\ndet_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)\ndet_count = np.where(det_class_ids == 0)[0][0]\ndet_class_ids = det_class_ids[:det_count]\ndetections = mrcnn['detections'][0, :det_count]\n\nprint(\"{} detections: {}\".format(\n    det_count, np.array(dataset.class_names)[det_class_ids]))\n\ncaptions = [\"{} {:.3f}\".format(dataset.class_names[int(c)], s) if c > 0 else \"\"\n            for c, s in zip(detections[:, 4], detections[:, 5])]\nvisualize.draw_boxes(\n    image, \n    refined_boxes=utils.denorm_boxes(detections[:, :4], image.shape[:2]),\n    visibilities=[2] * len(detections),\n    captions=captions, title=\"Detections\",\n    ax=get_ax())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.c Step by Step Detection\n\nHere we dive deeper into the process of processing the detections."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Proposals are in normalized coordinates\nproposals = mrcnn[\"proposals\"][0]\n\n# Class ID, score, and mask per proposal\nroi_class_ids = np.argmax(mrcnn[\"probs\"][0], axis=1)\nroi_scores = mrcnn[\"probs\"][0, np.arange(roi_class_ids.shape[0]), roi_class_ids]\nroi_class_names = np.array(dataset_val.class_names)[roi_class_ids]\nroi_positive_ixs = np.where(roi_class_ids > 0)[0]\n\n# How many ROIs vs empty rows?\nprint(\"{} Valid proposals out of {}\".format(np.sum(np.any(proposals, axis=1)), proposals.shape[0]))\nprint(\"{} Positive ROIs\".format(len(roi_positive_ixs)))\n\n# Class counts\nprint(list(zip(*np.unique(roi_class_names, return_counts=True))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display a random sample of proposals.\n# Proposals classified as background are dotted, and\n# the rest show their class and confidence score.\nlimit = 200\nixs = np.random.randint(0, proposals.shape[0], limit)\ncaptions = [\"{} {:.3f}\".format(dataset_val.class_names[c], s) if c > 0 else \"\"\n            for c, s in zip(roi_class_ids[ixs], roi_scores[ixs])]\nvisualize.draw_boxes(\n    image,\n    boxes=utils.denorm_boxes(proposals[ixs], image.shape[:2]),\n    visibilities=np.where(roi_class_ids[ixs] > 0, 2, 1),\n    captions=captions, title=\"ROIs Before Refinement\",\n    ax=get_ax())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Apply Bounding Box Refinement"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Class-specific bounding box shifts.\nroi_bbox_specific = mrcnn[\"deltas\"][0, np.arange(proposals.shape[0]), roi_class_ids]\nlog(\"roi_bbox_specific\", roi_bbox_specific)\n\n# Apply bounding box transformations\n# Shape: [N, (y1, x1, y2, x2)]\nrefined_proposals = utils.apply_box_deltas(\n    proposals, roi_bbox_specific * config.BBOX_STD_DEV)\nlog(\"refined_proposals\", refined_proposals)\n\n# Show positive proposals\n# ids = np.arange(roi_boxes.shape[0])  # Display all\nlimit = 5\nids = np.random.randint(0, max(len(roi_positive_ixs),1), limit)  # Display random sample\ncaptions = [\"{} {:.3f}\".format(dataset_val.class_names[c], s) if c > 0 else \"\"\n            for c, s in zip(roi_class_ids[roi_positive_ixs][ids], roi_scores[roi_positive_ixs][ids])]\nvisualize.draw_boxes(\n    image, ax=get_ax(),\n    boxes=utils.denorm_boxes(proposals[roi_positive_ixs][ids], image.shape[:2]),\n    refined_boxes=utils.denorm_boxes(refined_proposals[roi_positive_ixs][ids], image.shape[:2]),\n    visibilities=np.where(roi_class_ids[roi_positive_ixs][ids] > 0, 1, 0),\n    captions=captions, title=\"ROIs After Refinement\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Filter Low Confidence Detections"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove boxes classified as background\nkeep = np.where(roi_class_ids > 0)[0]\nprint(\"Keep {} detections:\\n{}\".format(keep.shape[0], keep))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove low confidence detections\nkeep = np.intersect1d(keep, np.where(roi_scores >= config.DETECTION_MIN_CONFIDENCE)[0])\nprint(\"Remove boxes below {} confidence. Keep {}:\\n{}\".format(\n    config.DETECTION_MIN_CONFIDENCE, keep.shape[0], keep))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Per-Class Non-Max Suppression"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Apply per-class non-max suppression\npre_nms_boxes = refined_proposals[keep]\npre_nms_scores = roi_scores[keep]\npre_nms_class_ids = roi_class_ids[keep]\n\nnms_keep = []\nfor class_id in np.unique(pre_nms_class_ids):\n    # Pick detections of this class\n    ixs = np.where(pre_nms_class_ids == class_id)[0]\n    # Apply NMS\n    class_keep = utils.non_max_suppression(pre_nms_boxes[ixs], \n                                            pre_nms_scores[ixs],\n                                            config.DETECTION_NMS_THRESHOLD)\n    # Map indicies\n    class_keep = keep[ixs[class_keep]]\n    nms_keep = np.union1d(nms_keep, class_keep)\n    print(\"{:22}: {} -> {}\".format(dataset_val.class_names[class_id][:20], \n                                   keep[ixs], class_keep))\n\nkeep = np.intersect1d(keep, nms_keep).astype(np.int32)\nprint(\"\\nKept after per-class NMS: {}\\n{}\".format(keep.shape[0], keep))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show final detections\nixs = np.arange(len(keep))  # Display all\n# ixs = np.random.randint(0, len(keep), 10)  # Display random sample\ncaptions = [\"{} {:.3f}\".format(dataset_val.class_names[c], s) if c > 0 else \"\"\n            for c, s in zip(roi_class_ids[keep][ixs], roi_scores[keep][ixs])]\nvisualize.draw_boxes(\n    image,\n    boxes=utils.denorm_boxes(proposals[keep][ixs], image.shape[:2]),\n    refined_boxes=utils.denorm_boxes(refined_proposals[keep][ixs], image.shape[:2]),\n    visibilities=np.where(roi_class_ids[keep][ixs] > 0, 1, 0),\n    captions=captions, title=\"Detections after NMS\",\n    ax=get_ax())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stage 3: Generating Masks\n\nThis stage takes the detections (refined bounding boxes and class IDs) from the previous layer and runs the mask head to generate segmentation masks for every instance."},{"metadata":{},"cell_type":"markdown","source":"### 3.a Mask Targets\n\nThese are the training targets for the mask branch"},{"metadata":{"trusted":true},"cell_type":"code","source":"limit = 8\ndisplay_images(np.transpose(gt_mask[..., :limit], [2, 0, 1]), cmap=\"Blues\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.b Predicted Masks"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get predictions of mask head\nmrcnn = model_detect.run_graph([image], [\n    (\"detections\", model_detect.keras_model.get_layer(\"mrcnn_detection\").output),\n    (\"masks\",model_detect.keras_model.get_layer(\"mrcnn_mask\").output),\n])\n\n# Get detection class IDs. Trim zero padding.\ndet_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)\ndet_count = np.where(det_class_ids == 0)[0][0]\ndet_class_ids = det_class_ids[:det_count]\n\nprint(\"{} detections: {}\".format(\n    det_count, np.array(dataset.class_names)[det_class_ids]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Masks\ndet_boxes = utils.denorm_boxes(mrcnn[\"detections\"][0, :, :4], image.shape[:2])\ndet_mask_specific = np.array([mrcnn[\"masks\"][0, i, :, :, c] \n                              for i, c in enumerate(det_class_ids)])\ndet_masks = np.array([utils.unmold_mask(m, det_boxes[i], image.shape)\n                      for i, m in enumerate(det_mask_specific)])\nlog(\"det_mask_specific\", det_mask_specific)\nlog(\"det_masks\", det_masks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_images(det_mask_specific[:4] * 255, cmap=\"Blues\", interpolation=\"none\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_images(det_masks[:4] * 255, cmap=\"Blues\", interpolation=\"none\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize Activations\n\nIn some cases it helps to look at the output from different layers and visualize them to catch issues and odd patterns."},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"# Get activations of a few sample layers\nactivations = model_detect.run_graph([image], [\n    (\"input_image\",        tf.identity(model_detect.keras_model.get_layer(\"input_image\").output)),\n    (\"res2c_out\",          model_detect.keras_model.get_layer(\"res2c_out\").output),\n    (\"res3c_out\",          model_detect.keras_model.get_layer(\"res3c_out\").output),\n    (\"rpn_bbox\",           model_detect.keras_model.get_layer(\"rpn_bbox\").output),\n    (\"roi\",                model_detect.keras_model.get_layer(\"ROI\").output),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Backbone feature map\ndisplay_images(np.transpose(activations[\"res2c_out\"][0,:,:,:4], [2, 0, 1]), cols=4)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}