{"metadata": {"language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "name": "python", "nbconvert_exporter": "python", "mimetype": "text/x-python", "version": "3.6.4", "pygments_lexer": "ipython3"}, "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}, "cells": [{"source": ["## Some code to save you time\n", "\n", "A slight improvement on 'rle_encode' from my kernel https://www.kaggle.com/stainsby/fast-tested-rle used in the Carvana Image Masking Challenge. This kernel also reads all stage 1 test and training data into memory.\n", "\n", "The RLE encoding routine is tested by:\n", "\n", "1. RLE encoding all masks and then decoding them and checking that the output matches the input mask.\n", "2. Checking the generated RLEs against the supplied data in `stage1_train_labels.csv`.\n", "\n", "Note that there was a issue with an earlier version encoding in the wrong direction that has now been fixed. Thanks to [Lam Dang](https://www.kaggle.com/lamdang) for pointing this out. **This may depend on the library that you use to read image files, so be aware that you will need to preprocess your mask with a transpose operation  if your library reads images as width \u00d7 height instead of height \u00d7 width** (thanks to [firolino](https://www.kaggle.com/firolino))."], "metadata": {"_uuid": "0270f35c2cdacbf6ee016afdb146d82f37327a2c", "_cell_guid": "afa39594-ff66-4fcb-a830-e845cfb0d889"}, "cell_type": "markdown"}, {"source": ["import os, time\n", "from pathlib import Path\n", "\n", "import numpy as np\n", "\n", "import imageio\n", "import matplotlib.pyplot as plt"], "metadata": {"_uuid": "48a16079268e7071b2dcedb32b5c23b6c1d8c445", "collapsed": true, "_cell_guid": "3f9fd777-3dad-43ac-b76b-4aca1eecfca0"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": ["WORKING_DIR = Path(os.getcwd())\n", "PROJECT_DIR = WORKING_DIR.parent\n", "INPUT_DIR = PROJECT_DIR / 'input'\n", "TRAIN_DIR = INPUT_DIR / 'stage1_train'\n", "TEST_DIR = INPUT_DIR / 'stage1_test'\n", "\n", "# Images with errors that should be skipped. Adjust as required.\n", "TRAIN_ERROR_IDS = [\n", "    '7b38c9173ebe69b4c6ba7e703c0c27f39305d9b2910f46405993d2ea7a963b80'\n", "]"], "metadata": {"_uuid": "f6c6b57ba74fdc29629a86af8e31c0532db966a7", "collapsed": true, "_cell_guid": "1bae27a9-de5d-4767-826a-7cfc73d9a2a4"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": ["## Data input routines"], "metadata": {"_uuid": "b18102f03e106919c69f0ec327589864f9b343c1", "_cell_guid": "a219c71f-c121-4678-adfc-54e58bf4a7a9"}, "cell_type": "markdown"}, {"source": ["def image_ids_in(root_dir, is_train_data=False):\n", "    ids = []\n", "    for id in os.listdir(root_dir):\n", "        if id in TRAIN_ERROR_IDS:\n", "            print('Skipping ID due to bad training data:', id)\n", "        else:\n", "            ids.append(id)\n", "    return ids\n", "\n", "TRAIN_IMAGE_IDS = image_ids_in(TRAIN_DIR, is_train_data=True)\n", "TEST_IMAGE_IDS = image_ids_in(TEST_DIR)\n", "\n", "print('Examples:', TRAIN_IMAGE_IDS[22], TEST_IMAGE_IDS[22])"], "metadata": {"_uuid": "0f996481649c8e4f436ff37578c7793847f1c789", "collapsed": true, "_cell_guid": "79583dc8-b26e-4193-842d-c6d4c1326d82"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": ["def load_images(root_dir, ids, get_masks=False):\n", "    images = []\n", "    masks = []\n", "    image_sizes = []\n", "    for id in ids:\n", "        item_dir = root_dir / id\n", "        image_path = item_dir / 'images' / (id + '.png')\n", "        image = imageio.imread(str(image_path))\n", "        image = image[:, :, :3] # remove the alpha channel as it is not used\n", "        images.append(image)\n", "        image_sizes.append(image.shape[:2])\n", "        if get_masks:\n", "            mask_sequence = []\n", "            masks_dir = item_dir / 'masks'\n", "            mask_paths = masks_dir.glob('*.png')\n", "            for mask_path in mask_paths:\n", "                mask = imageio.imread(str(mask_path)) # 0 and 255 values\n", "                mask = (mask > 0).astype(np.uint8) # 0 and 1 values\n", "                mask_sequence.append(mask)\n", "            masks.append(mask_sequence)\n", "    if get_masks:\n", "        return images, masks, image_sizes\n", "    else:\n", "        return images, image_sizes\n", "\n", "TRAIN_IMAGES, TRAIN_MASKS, TRAIN_IMAGE_SIZES = load_images(TRAIN_DIR, TRAIN_IMAGE_IDS, True)\n", "TEST_IMAGES, TEST_IMAGE_SIZES  = load_images(TEST_DIR, TEST_IMAGE_IDS, False)"], "metadata": {"_uuid": "c87e3a127c7c1a6700eb67359dce69066537d70a", "collapsed": true, "_cell_guid": "7c4eb684-68df-4bd6-8d96-d83014367681"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": ["def show_image_shape_stats(image_sizes, image_ids):\n", "    print('  no. of images:', len(image_sizes))\n", "    print('  first five shapes:', image_sizes[:5])\n", "    image_sizes = np.asarray(image_sizes)\n", "    image_ids = np.asarray(image_ids)\n", "    print('  min. width:', image_sizes[:, 1].min(), '; max width:', image_sizes[:, 1].max())\n", "    print('  min. height:', image_sizes[:, 0].min(), '; max height:', image_sizes[:, 0].max())\n", "    pixel_counts = image_sizes.prod(axis=1)\n", "    sorted_pixel_count_indices = np.argsort(pixel_counts)[::-1] # biggest to smallest\n", "    print('  biggest images (by pixel count):\\n   ', \\\n", "          '\\n    '.join(image_ids[sorted_pixel_count_indices[:3]]))\n", "    print('  smallest images (by pixel count):\\n   ', \\\n", "          '\\n    '.join(image_ids[sorted_pixel_count_indices[-3:]]))\n", "\n", "print('Train image stats:')\n", "show_image_shape_stats(TRAIN_IMAGE_SIZES, TRAIN_IMAGE_IDS)\n", "print('\\nTest image stats:')\n", "show_image_shape_stats(TEST_IMAGE_SIZES, TEST_IMAGE_IDS)"], "metadata": {"_uuid": "b9bb41c1d697691603b224441102118d49326eb1", "collapsed": true, "_cell_guid": "8e4e6400-47f7-4b5b-938b-9d961f1fe942"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": ["## RLE routines"], "metadata": {"_uuid": "34e81f235f968ce37843e8dc60cc88b09035c573", "_cell_guid": "80012b2e-a182-42ad-a1c7-307ff37e1ec4"}, "cell_type": "markdown"}, {"source": ["def rle_encode(mask):\n", "    pixels = mask.T.flatten()\n", "    # We need to allow for cases where there is a '1' at either end of the sequence.\n", "    # We do this by padding with a zero at each end when needed.\n", "    use_padding = False\n", "    if pixels[0] or pixels[-1]:\n", "        use_padding = True\n", "        pixel_padded = np.zeros([len(pixels) + 2], dtype=pixels.dtype)\n", "        pixel_padded[1:-1] = pixels\n", "        pixels = pixel_padded\n", "    rle = np.where(pixels[1:] != pixels[:-1])[0] + 2\n", "    if use_padding:\n", "        rle = rle - 1\n", "    rle[1::2] = rle[1::2] - rle[:-1:2]\n", "    return rle\n", "\n", "\n", "def rle_to_string(runs):\n", "    return ' '.join(str(x) for x in runs)\n", "\n", "\n", "# Used only for testing.\n", "# This is copied from https://www.kaggle.com/paulorzp/run-length-encode-and-decode.\n", "# Thanks to Paulo Pinto.\n", "def rle_decode(rle_str, mask_shape, mask_dtype):\n", "    s = rle_str.split()\n", "    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n", "    starts -= 1\n", "    ends = starts + lengths\n", "    mask = np.zeros(np.prod(mask_shape), dtype=mask_dtype)\n", "    for lo, hi in zip(starts, ends):\n", "        mask[lo:hi] = 1\n", "    return mask.reshape(mask_shape[::-1]).T"], "metadata": {"_uuid": "bcaba059a6749073d4fe532b563d3d9deee16418", "collapsed": true, "_cell_guid": "886366b6-170a-4612-8aca-0fd4012af36a"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": ["### Tests"], "metadata": {"_uuid": "32254608621fa5ccbbbb9f29ecd2f872eb910a5e", "_cell_guid": "ab09fc11-6b54-42bf-b13b-395f0f2b6621"}, "cell_type": "markdown"}, {"source": ["# Used for testing only.\n", "def read_sample_training_rles():\n", "    is_header = True\n", "    id_rle_map = {}\n", "    for line in open(INPUT_DIR / 'stage1_train_labels.csv').readlines():\n", "        if is_header:\n", "            is_header = False\n", "            continue\n", "        id, rle = line.split(',')\n", "        id = id.strip()\n", "        rle = rle.strip()\n", "        assert len(id) > 0\n", "        assert len(rle) > 0\n", "        if id in TRAIN_ERROR_IDS:\n", "            continue\n", "        rles = id_rle_map.get(id)\n", "        if not rles:\n", "            rles = set([])\n", "            id_rle_map[id] = rles\n", "        rles.add(rle)\n", "    ids = id_rle_map.keys()\n", "    print('Read sample RLEs for', len(ids), 'images')\n", "    return id_rle_map\n", "    \n", "\n", "SAMPLE_TRAINING_RLES = read_sample_training_rles()\n", "\n", "\n", "def test_rle_encode():\n", "    test_mask = np.asarray([[0, 0, 0, 0], [0, 0, 1, 1], [0, 0, 1, 1], [1, 0, 0, 0]]).T\n", "    assert rle_to_string(rle_encode(test_mask)) == '7 2 11 3'\n", "    test_mask = np.asarray([[0, 0, 0, 0], [0, 0, 1, 1], [0, 0, 1, 1], [1, 0, 0, 1]]).T\n", "    assert rle_to_string(rle_encode(test_mask)) == '7 2 11 3 16 1'\n", "    test_mask = np.asarray([[1, 0, 0, 0], [0, 0, 1, 1], [0, 0, 1, 1], [1, 0, 0, 0]]).T\n", "    assert rle_to_string(rle_encode(test_mask)) == '1 1 7 2 11 3'\n", "    test_mask = np.asarray([[1, 0, 0, 0], [0, 0, 1, 1], [0, 0, 1, 1], [1, 0, 0, 1]]).T\n", "    assert rle_to_string(rle_encode(test_mask)) == '1 1 7 2 11 3 16 1'\n", "    num_images = len(TRAIN_IMAGES)\n", "    print('Verfiying RLE encoding on', num_images, 'mask sequences ...')\n", "    time_rle = 0.0 # seconds\n", "    time_stringify = 0.0 # seconds\n", "    mask_count = 0\n", "    for image_idx in range(num_images):\n", "        image_id = TRAIN_IMAGE_IDS[image_idx]\n", "        mask_sequence = TRAIN_MASKS[image_idx]\n", "        num_masks = len(mask_sequence)\n", "        sample_rles = SAMPLE_TRAINING_RLES[image_id]\n", "        assert num_masks == len(sample_rles), \\\n", "                'number of masks should match that of RLEs in supplied sample'\n", "        for mask_idx in range(num_masks):\n", "            mask = mask_sequence[mask_idx]\n", "            t0 = time.clock()\n", "            rle = rle_encode(mask)\n", "            assert len(rle) % 2 == 0 , 'RLE array length should be even'\n", "            time_rle += time.clock() - t0\n", "            t0 = time.clock()\n", "            rle_str = rle_to_string(rle)\n", "            time_stringify += time.clock() - t0\n", "            assert rle_str in sample_rles, 'RLE not found in supplied sample'\n", "            regenerated_mask = rle_decode(rle_str, mask.shape, mask.dtype)\n", "            assert mask.dtype == regenerated_mask.dtype\n", "            assert np.array_equal(mask.shape, regenerated_mask.shape), \\\n", "                    repr(mask.shape) + ' v. ' + repr(regenerated_mask.shape)\n", "            assert np.array_equal(mask, regenerated_mask), \\\n", "                    'mask does not match regenerated mask'\n", "            if mask_count and (mask_count % 5000) == 0:\n", "                print('  ..', mask_count, 'masks tested ..')\n", "            mask_count += 1\n", "    print('Total number of masks encoded:', mask_count)\n", "    print('Time spent RLE encoding masks:', time_rle, 's =>', \\\n", "            1000*(time_rle/mask_count), 'ms per mask.')\n", "    print('Time spent stringifying RLEs:', time_stringify, 's =>', \\\n", "            1000*(time_stringify/mask_count), 'ms per mask.')\n", "\n", "\n", "test_rle_encode()"], "metadata": {"_uuid": "4484c1de12cfc482b8251ac6608c294e1a532911", "collapsed": true, "_cell_guid": "5351960b-714e-4269-af8e-f5f4a0a13d15"}, "cell_type": "code", "execution_count": null, "outputs": []}, {"source": [], "metadata": {"_uuid": "1faffd9521cf8b0ec1a21171af5f239b22ce9b29", "collapsed": true, "_cell_guid": "e7bd9c96-0274-417d-9f97-4ca9ad6849e6"}, "cell_type": "code", "execution_count": null, "outputs": []}], "nbformat_minor": 1, "nbformat": 4}