{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"Currently we've many models around with different performances. For model without any post-processing we can expect:\n* LinkNet: Around LB 0.36\n* UNet: Around LB 0.37 to 0.43\n* Mask RCNN: Around LB 0.45 to 0.50\n\nPredictions to deliver for each image for is a list of masks detected (the instances). Now, how can we ensemble models ouputs to get better performances? For models that provide per pixel probabilities (LinkNet, UNet) one can simply do averaging/stacking on the probabilities. But for models that provide instances (with or without scoring), what can be done? This kernel is to try to experiment solutions to ensemble instances.\n\n2018-03-20: First, we apply really basic NMS (Non-Maximum-Suppression) with bounding boxes extracted from predicted masks. 2 predictions provided as a starting point:\n* Model#0: LB 0.421 (Mask RCNN - Resnet101 backbone, pretrained - Coco, CV fold1)\n* Model#1: LB 0.413 (Mask RCNN - Resnet101 backbone, pretrained - Coco, CV fold2)\n\nForks and contributions welcome!\n"},{"metadata":{"_cell_guid":"33ac41bc-1816-4948-b8c8-961712cae6bd","_uuid":"de29f6e4972faaa618b72fbdb01c642fcb1c7334","collapsed":true,"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport skimage.io\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\n# from subprocess import check_output\n# print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"STAGE1_TEST = \"../input/data-science-bowl-2018/stage1_test\"\nSTAGE1_TEST_IMAGE_PATTERN = \"%s/{}/images/{}.png\" % STAGE1_TEST\nSUBMISSION_IMAGEID = \"ImageId\"\nSUBMISSION_ENCODED = \"EncodedPixels\"\nmodels_path = [\n    \"../input/lb0421/submission_maskrcnn_0.421.csv\",\n    \"../input/lb0413/submission_maskrcnn_0.413.csv\"\n]","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"231fcdd5-4146-4c50-8f65-0723384f0378","_uuid":"649d78088a207ac9d912c493610692a2feae9679","collapsed":true,"trusted":true},"cell_type":"code","source":"# Image loading\ndef image_ids_in(root_dir):\n    ids = []\n    for id in os.listdir(root_dir):\n        ids.append(id)\n    return ids\n\ndef read_image(image_id, pattern=STAGE1_TEST_IMAGE_PATTERN):\n    image_file = pattern.format(image_id, image_id)\n    image = skimage.io.imread(image_file)\n    # Drop alpha which is not used\n    image = image[:, :, :3]\n    return image\n\ndef image_id_to_index(image_id, images_ids):\n    i = np.argwhere(np.array(images_ids) == image_id)[0][0]\n    return i","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"eb12265b-4281-4640-b544-242853b90bb4","_uuid":"5e7151e99f82c544ba560b3cf42aff1065d90eb6","collapsed":true,"trusted":true},"cell_type":"code","source":"# RLE decoding functions\ndef rle_decode_one_mask(rle_str, mask_shape, mask_dtype):\n    s = rle_str.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    mask = np.zeros(np.prod(mask_shape), dtype=mask_dtype)\n    for lo, hi in zip(starts, ends):\n        mask[lo:hi] = 1\n    return mask.reshape(mask_shape[::-1]).T\n\ndef rle_decode_all_masks(masks_str, mask_shape, mask_dtype):\n    image = None\n    i = 0\n    for mask_str in masks_str:\n        i = i + 1\n        mask = rle_decode_one_mask(mask_str, mask_shape, mask_dtype)\n        mask[mask == 1] = i\n        if image is None:\n            image = mask\n        else:\n            image = image + mask\n    return image","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"bb154f1c-345d-4f54-8ab6-3c5a9f508798","_uuid":"74466816c3589f35a956fadeed965fac7911c52d","collapsed":true,"trusted":true},"cell_type":"code","source":"# Test images\ntest_image_ids = image_ids_in(STAGE1_TEST)","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"aceeddb8-dc99-487a-91e6-c3bac5712c7a","_uuid":"6a194936447b5c152c007d4f32a270ce3bb01759","collapsed":true,"trusted":true},"cell_type":"code","source":"# Convert index image (unique value per mask) to array.\ndef img_masks_as_masks_array(train_label):\n    # As (masks, height, width)\n    y_true = []\n    uniques = np.unique(train_label)\n    # Remove zero from index\n    indexes = np.delete(uniques, np.where(uniques == 0))\n    for index in indexes:\n        y_true.append(np.where(train_label == index, 1, 0))\n    y_true = np.array(y_true)\n    return y_true\n\n# Convert back all mask to index image\ndef masks_array_to_index_image(masks):\n    mask = np.zeros((masks.shape[1], masks.shape[2]), dtype=np.uint16)\n    for index in range(0, masks.shape[0]):\n        mask[masks[index,:,:] > 0] = index + 1\n    return mask\n\n# Read image and predicted masks\ndef read_test_image_mask(submissionPD, test_id):\n    test_image = read_image(test_id)\n    rle_encoded_masks = submissionPD[submissionPD[SUBMISSION_IMAGEID] == test_id][SUBMISSION_ENCODED].values\n    test_masks = rle_decode_all_masks(rle_encoded_masks, test_image.shape[:-1], np.int32)\n    test_masks_array = img_masks_as_masks_array(test_masks)\n    return test_image, test_masks_array","execution_count":24,"outputs":[]},{"metadata":{"_cell_guid":"74977619-3464-4fee-a110-557f3ad35292","_uuid":"7c2dc4131f000c4cf9ba6a771b233b0121b82bb8","collapsed":true,"trusted":true},"cell_type":"code","source":"# Extract bounding box of mask\ndef find_bounding_boxes_on_mask(bin_img, test_id, mask_id, with_ref=None):\n    boxes = []\n    img_bin = np.where(bin_img > 0, 1, 0)\n    img_rgb = (img_bin)*255\n    img_rgb = np.concatenate([img_rgb[:, :, np.newaxis], img_rgb[:, :, np.newaxis], img_rgb[:, :, np.newaxis]], axis=-1)\n    img_rgb = img_rgb.astype(np.uint8)\n    im_bw = cv2.cvtColor(img_rgb,cv2.COLOR_RGB2GRAY)\n    ret, im_bw = cv2.threshold(im_bw, 127, 255, cv2.THRESH_BINARY)\n    pixelpoints = cv2.findNonZero(im_bw)\n    x, y, w, h = cv2.boundingRect(pixelpoints)\n    if with_ref is not None:\n        boxes.append((x, y, w, h, with_ref, test_id, mask_id))\n    else:\n        boxes.append((x,y,w,h))\n    return np.array(boxes)","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"a4d582fb-c862-44ba-8200-80235599e7e3","_uuid":"9b696e71a72149dea7733cdd326e2d78a92f4fa3","collapsed":true,"trusted":true},"cell_type":"code","source":"# Extract all bounding boxes\ndef find_bounding_boxes_on_masks(test_masks_array, test_id, with_ref=None):\n    test_masks_pass = []\n    boxes_masks = []\n    for mask_id in range(0, len(test_masks_array)):\n        mask = test_masks_array[mask_id]\n        boxes = find_bounding_boxes_on_mask(mask, test_id, mask_id, with_ref=with_ref)\n        boxes_masks.append(boxes)\n        test_masks_pass.append(mask)\n    test_masks_pass = np.array(test_masks_pass)\n    boxes_masks = np.array(boxes_masks)\n    return test_masks_pass, boxes_masks","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"64dddc99-8f35-4111-9de1-9a2075077f3a","_uuid":"c655b1ef06c945552164cc998b6b4a141fe964f4","collapsed":true,"trusted":true},"cell_type":"code","source":"# Image and array of masks + bounding boxes for each model (for a given image).\ndef models_cv_masks_for_image(models_path, test_id, test_image_ids):\n    test_id_ref = image_id_to_index(test_id, test_image_ids)\n    models_cv_masks = []\n    models_cv_masks_boxes = []\n    for i in range(0, len(models_path)):\n        model_path = models_path[i]\n        submission = pd.read_csv(model_path)\n        submission.dropna(subset=[SUBMISSION_ENCODED], inplace=True)\n        test_image, test_masks_array = read_test_image_mask(submission, test_id)\n        test_masks_clean, boxes_masks = find_bounding_boxes_on_masks(test_masks_array, test_id_ref, with_ref=i)\n        models_cv_masks.append(test_masks_clean)\n        models_cv_masks_boxes.append(boxes_masks)\n    return test_image, models_cv_masks, models_cv_masks_boxes","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"a50a53b7-9050-4843-978c-35aa7e37c014","_uuid":"309ad28405968246baed0781ea7b08d2af4733d9","collapsed":true,"trusted":true},"cell_type":"code","source":"# Basic NMS on boxes, https://www.pyimagesearch.com/2014/11/17/non-maximum-suppression-object-detection-python\n# Malisiewicz et al.\ndef non_max_suppression_fast(boxes, overlapThresh):\n    # if there are no boxes, return an empty list\n    if len(boxes) == 0:\n        return []\n\n    # if the bounding boxes integers, convert them to floats --\n    # this is important since we'll be doing a bunch of divisions\n    if boxes.dtype.kind == \"i\":\n        boxes = boxes.astype(\"float\")\n\n    # initialize the list of picked indexes\n    pick = []\n\n    # grab the coordinates of the bounding boxes\n    x1 = boxes[:,0]\n    y1 = boxes[:,1]\n    x2 = boxes[:,0] + boxes[:,2]\n    y2 = boxes[:,1] + boxes[:,3]\n    # compute the area of the bounding boxes and sort the bounding\n    # boxes by the bottom-right y-coordinate of the bounding box\n    area = (x2 - x1 + 1) * (y2 - y1 + 1)\n    idxs = np.argsort(y2)\n\n    # keep looping while some indexes still remain in the indexes list\n    while len(idxs) > 0:\n        # grab the last index in the indexes list and add the\n        # index value to the list of picked indexes\n        last = len(idxs) - 1\n        i = idxs[last]\n        pick.append(i)\n\n        # find the largest (x, y) coordinates for the start of the bounding box and the smallest (x, y) coordinates for the end of the bounding box\n        xx1 = np.maximum(x1[i], x1[idxs[:last]])\n        yy1 = np.maximum(y1[i], y1[idxs[:last]])\n        xx2 = np.minimum(x2[i], x2[idxs[:last]])\n        yy2 = np.minimum(y2[i], y2[idxs[:last]])\n        # compute the width and height of the bounding box\n        w = np.maximum(0, xx2 - xx1 + 1)\n        h = np.maximum(0, yy2 - yy1 + 1)\n        # compute the ratio of overlap\n        overlap = (w * h) / area[idxs[:last]]\n        # delete all indexes from the index list that have\n        idxs = np.delete(idxs, np.concatenate(([last],\n            np.where(overlap > overlapThresh)[0])))\n    # return only the bounding boxes that were picked using the integer data type\n    return boxes[pick].astype(\"int\")","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"c03db6bd-b2b9-4502-afb6-5aea1b05c248","_uuid":"dffcfeac39ba433748f7a94b8dd827d87a9c16ce","collapsed":true,"trusted":true},"cell_type":"code","source":"# Compute NMS (i.e. select only one box when multiple boxes overlap) for across models.\ndef models_cv_masks_boxes_nms(models_cv_masks_boxes, threshold=0.3):\n    boxes = np.concatenate(models_cv_masks_boxes).squeeze()\n    boxes_nms = non_max_suppression_fast(boxes, threshold)\n    return boxes_nms","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"d9825eb6-7b07-4b2b-a242-b62ee1332621","_uuid":"ce10e1d76d34e77ee1714d4929585cc73334f366","collapsed":true,"trusted":true},"cell_type":"code","source":"# Display some result (on the nightmare images)\n# test_id = \"0f1f896d9ae5a04752d3239c690402c022db4d72c0d2c087d73380896f72c466\"\ntest_id = \"472b1c5ff988dadc209faea92499bc07f305208dbda29d16262b3d543ac91c71\"","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"64f1c8ca-bf9a-4b53-8e0d-142d6d5f55ca","_uuid":"a2ff08f2eb54f1b7ac1ed820167cdb956abf6bf8","collapsed":true,"scrolled":true,"trusted":true},"cell_type":"code","source":"# Get masks and boxes (one per mask) for each model\ntest_image, test_masks_cv_array, test_masks_boxes_cv_array = models_cv_masks_for_image(models_path, test_id, test_image_ids)","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"fe60ab28-b8ba-4ef6-9523-2b34435f84d7","_uuid":"c5960cd06e4ea4ebc4c243faf65664cf686c36fd","collapsed":true,"trusted":true},"cell_type":"code","source":"# Run NMS ensembling\nmasks_boxes_nms = models_cv_masks_boxes_nms(test_masks_boxes_cv_array, threshold=0.3)","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"d2189ef2-19ac-483b-be45-b28c3e2ba07c","_uuid":"d6ba88a28cfdaf96f32a33c5bc3f1c1d26991e1b","scrolled":false,"trusted":true},"cell_type":"code","source":"# Plot predictions of each model\nfig, ax = plt.subplots(1, 2, figsize=(18, 8))\nax[0].axis('off')\nax[0].imshow(masks_array_to_index_image(test_masks_cv_array[0]), cmap='nipy_spectral')\nax[0].imshow(test_image, alpha=0.45)\nax[0].set_title(\"Model#0: %d predicted instances for %s\"%(len(test_masks_cv_array[0]), models_path[0]))\n\nax[1].axis('off')\nax[1].imshow(masks_array_to_index_image(test_masks_cv_array[1]), cmap='nipy_spectral')\nax[1].imshow(test_image, alpha=0.45)\nax[1].set_title(\"Model#1: %d predicted instances for %s\"%(len(test_masks_cv_array[1]), models_path[1]))\n\nplt.tight_layout()","execution_count":37,"outputs":[]},{"metadata":{"_cell_guid":"664248cd-3252-4eca-855e-6f33b3172936","_uuid":"0514017360ae4ce7ec544d6cf4f32fa72d46d2f4","trusted":true},"cell_type":"code","source":"# Plot boxes for each model (left) and resulting NMS (right)\nfig, ax = plt.subplots(1, 2, figsize=(18, 8))\nax[0].axis('off')\nax[0].set_ylim(test_image.shape[0] + 10, -10)\nax[0].set_xlim(-10, test_image.shape[1] + 10)\ncmap = plt.cm.get_cmap('nipy_spectral')\n# Plot boxes per model\nfor box in np.concatenate(test_masks_boxes_cv_array).squeeze():\n    p = patches.Rectangle((box[0]-1, box[1]-1), box[2], box[3], linewidth=1, facecolor='none', edgecolor=cmap(box[4]*60), alpha=0.75, linestyle=\"dashed\")\n    ax[0].add_patch(p)\n    ax[0].text(box[0], box[1] + 8, \"%d\"%box[4], color=cmap(box[4]*60), size=10, backgroundcolor=\"none\") \nax[0].imshow(test_image, alpha=0.6)\nax[0].set_title(\"Bounding boxes of predicted instances for model #0 and #1\")\n\n# Plot NMS results\nax[1].set_ylim(test_image.shape[0] + 10, -10)\nax[1].set_xlim(-10, test_image.shape[1] + 10)\nax[1].axis('off')\nfor box_nms in masks_boxes_nms:\n    p = patches.Rectangle((box_nms[0]-1, box_nms[1]-1), box_nms[2], box_nms[3], linewidth=1, facecolor='yellow', alpha=0.25, linestyle=\"dashed\")\n    ax[1].add_patch(p)\n    ax[1].text(box_nms[0], box_nms[1] + 8, \"%d\"%box_nms[4], color=cmap(box_nms[4]*60), size=11, backgroundcolor=\"none\")  \nax[1].imshow(test_image, alpha=0.6)\nax[1].set_title(\"Ensemble NMS bounding boxes (%d) of predicted instances with its reference model #0 or #1\"%len(masks_boxes_nms))\nplt.tight_layout()","execution_count":16,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"fd3e840d4a69f04844adfc2f72903e561f6bc7d2"},"cell_type":"code","source":"# Back to masks from NMS boxes\ndef get_masks_from_boxes_nms(masks_boxes_nms, test_masks_cv_array):\n    masks_nms = []\n    for box_nms in masks_boxes_nms:\n        model_id = box_nms[4]\n        mask_id = box_nms[6]\n        mask_nms = test_masks_cv_array[model_id][mask_id]\n        masks_nms.append(mask_nms)\n    masks_nms = np.array(masks_nms)\n    return masks_nms","execution_count":22,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"7287b9bc8f30fce31a1f5fc36fc63ef981b90789"},"cell_type":"code","source":"# NMS instances\nmasks_nms = get_masks_from_boxes_nms(masks_boxes_nms, test_masks_cv_array)","execution_count":23,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd74a23aa32f7b19ec4a810cdaf08c0404aac5f6"},"cell_type":"code","source":"# Plot masks from NMS boxes\nfig, ax = plt.subplots(1, 2, figsize=(18, 8))\nax[0].axis('off')\nmasks_nms_image = masks_array_to_index_image(masks_nms)\nax[0].imshow(test_image)\nax[0].set_title(\"%s\"%test_id)\nax[1].axis('off')\nax[1].imshow(masks_nms_image, cmap='nipy_spectral')\nax[1].imshow(test_image, alpha=0.45)\nax[1].set_title(\"Ensemble predicted instances (%d)\"%len(masks_nms))\nplt.tight_layout()","execution_count":38,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"432ac8df37f04d85334a6685f3a0dc821dfc17d3"},"cell_type":"code","source":"# RLE encoder\ndef rle_to_string(runs):\n    return ' '.join(str(x) for x in runs)\n\ndef rle_encode_one_mask(mask):\n    pixels = mask.T.flatten()\n    use_padding = False\n    if pixels[0] or pixels[-1]:\n        use_padding = True\n        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n        pixel_padded[1:-1] = pixels\n        pixels = pixel_padded\n    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    if use_padding:\n        rle = rle - 1\n    rle[1::2] = rle[1::2] - rle[:-1:2]\n    return rle\n\ndef rle_encode_all_masks(masks):\n    values=list(np.unique(masks))\n    values.remove(0)\n    RLEs=[]\n    for v in values:\n        mask = np.where(masks == v, 1, 0)\n        rle = rle_encode_one_mask(mask)\n        rle_str = rle_to_string(rle)\n        RLEs.append(rle_str)\n    return RLEs","execution_count":42,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"96c83cdf9aa8e49b3a5ec58af4655d00680b4cbc"},"cell_type":"code","source":"# Generate submission from NMS\ndef generate_test_submission(image_ids, models_path):\n    results = []\n    for image_id in image_ids:\n        test_image, test_masks_cv_array, test_masks_boxes_cv_array = models_cv_masks_for_image(models_path, image_id, image_ids)\n        masks_boxes_nms = models_cv_masks_boxes_nms(test_masks_boxes_cv_array, threshold=0.3)\n        masks_nms = masks_array_to_index_image(get_masks_from_boxes_nms(masks_boxes_nms, test_masks_cv_array))\n        rle_encoded_masks = rle_encode_all_masks(masks_nms)\n        for rle_encoded_mask in rle_encoded_masks:\n            info = (image_id, rle_encoded_mask)\n            results.append(info)\n    df = pd.DataFrame(results, columns=[SUBMISSION_IMAGEID, SUBMISSION_ENCODED])\n    return df","execution_count":43,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4168f9f5d3a99782ad2da9aaf81b025a3a5d16a0"},"cell_type":"code","source":"submissionPD = generate_test_submission(test_image_ids, models_path)\nsubmissionPD.head()","execution_count":45,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"758a10b8e91fcb150a9e0b552c89e62873f8c7fd"},"cell_type":"code","source":"submissionPD.to_csv(\"submission.csv\", index=False, sep=\",\")","execution_count":47,"outputs":[]},{"metadata":{"_uuid":"19b3708fe9f76d15851baadf1cc536d5f45492bb"},"cell_type":"markdown","source":"New submission from NMS does not improve LB score (0.419). So what could be the next steps?\n*  Play with overlap threshold\n* Keep or drop outliers? (single box with no overlap)\n* ?"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}