{"cells": [{"metadata": {"_cell_guid": "5cb12f78-e5a9-47d9-bcfd-ca322744c2ce", "_uuid": "2903ba531fb15d3943b80f28774e0d24d0377741"}, "source": ["This is an example notebook to demonstrate how the IoU metric works for a single image. Please note: this is not the official scoring implementation, but should work in the same manner."], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "1b889269-1027-4261-a3e9-54f75223571b", "collapsed": true, "_uuid": "3fd84142326a40f5f7ba094ce49c125d5a95837b"}, "source": ["%matplotlib inline\n", "import numpy as np\n", "import skimage.io\n", "import matplotlib.pyplot as plt\n", "import skimage.segmentation\n", "\n", "# Load a single image and its associated masks\n", "id = '0a7d30b252359a10fd298b638b90cb9ada3acced4e0c0e5a3692013f432ee4e9'\n", "file = \"../input/stage1_train/{}/images/{}.png\".format(id,id)\n", "masks = \"../input/stage1_train/{}/masks/*.png\".format(id)\n", "image = skimage.io.imread(file)\n", "masks = skimage.io.imread_collection(masks).concatenate()\n", "height, width, _ = image.shape\n", "num_masks = masks.shape[0]\n", "\n", "# Make a ground truth label image (pixel value is index of object label)\n", "labels = np.zeros((height, width), np.uint16)\n", "for index in range(0, num_masks):\n", "    labels[masks[index] > 0] = index + 1\n", "\n", "# Show label image\n", "fig = plt.figure()\n", "plt.imshow(image)\n", "plt.title(\"Original image\")\n", "fig = plt.figure()\n", "plt.imshow(labels)\n", "plt.title(\"Ground truth masks\")\n", "\n", "# Simulate an imperfect submission\n", "offset = 2 # offset pixels\n", "y_pred = labels[offset:, offset:]\n", "y_pred = np.pad(y_pred, ((0, offset), (0, offset)), mode=\"constant\")\n", "y_pred[y_pred == 20] = 0 # Remove one object\n", "y_pred, _, _ = skimage.segmentation.relabel_sequential(y_pred) # Relabel objects\n", "\n", "# Show simulated predictions\n", "fig = plt.figure()\n", "plt.imshow(y_pred)\n", "plt.title(\"Simulated imperfect submission\")\n", "\n", "# Compute number of objects\n", "true_objects = len(np.unique(labels))\n", "pred_objects = len(np.unique(y_pred))\n", "print(\"Number of true objects:\", true_objects)\n", "print(\"Number of predicted objects:\", pred_objects)\n", "\n", "# Compute intersection between all objects\n", "intersection = np.histogram2d(labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects))[0]\n", "\n", "# Compute areas (needed for finding the union between all objects)\n", "area_true = np.histogram(labels, bins = true_objects)[0]\n", "area_pred = np.histogram(y_pred, bins = pred_objects)[0]\n", "area_true = np.expand_dims(area_true, -1)\n", "area_pred = np.expand_dims(area_pred, 0)\n", "\n", "# Compute union\n", "union = area_true + area_pred - intersection\n", "\n", "# Exclude background from the analysis\n", "intersection = intersection[1:,1:]\n", "union = union[1:,1:]\n", "union[union == 0] = 1e-9\n", "\n", "# Compute the intersection over union\n", "iou = intersection / union\n", "\n", "# Precision helper function\n", "def precision_at(threshold, iou):\n", "    matches = iou > threshold\n", "    true_positives = np.sum(matches, axis=1) == 1   # Correct objects\n", "    false_positives = np.sum(matches, axis=0) == 0  # Missed objects\n", "    false_negatives = np.sum(matches, axis=1) == 0  # Extra objects\n", "    tp, fp, fn = np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)\n", "    return tp, fp, fn\n", "\n", "# Loop over IoU thresholds\n", "prec = []\n", "print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n", "for t in np.arange(0.5, 1.0, 0.05):\n", "    tp, fp, fn = precision_at(t, iou)\n", "    p = tp / (tp + fp + fn)\n", "    print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tp, fp, fn, p))\n", "    prec.append(p)\n", "print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"_cell_guid": "bc357ad2-d10c-4f5b-9f6a-55d47336779f", "collapsed": true, "_uuid": "542d2b226d90fc13cc852979f529962387873420"}, "source": [], "cell_type": "code", "execution_count": null, "outputs": []}], "metadata": {"kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}, "language_info": {"name": "python", "nbconvert_exporter": "python", "version": "3.6.4", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py"}}, "nbformat_minor": 1, "nbformat": 4}