{"cells": [{"cell_type": "markdown", "source": ["# Part 0 - Intro"], "metadata": {"_uuid": "bb9a5222d826d11f6ed4367e6e2b4b4a3d5a1edb", "_cell_guid": "da854b15-3d24-4a36-b3ed-3d488835cb2c"}}, {"cell_type": "markdown", "source": ["This notebook takes a lot of inspiration from:\n", "- https://www.kaggle.com/keegil/keras-u-net-starter-lb-0-277\n", "- https://www.kaggle.com/takuok/keras-generator-starter-lb-0-326/notebook\n", "- and many others!\n", "\n", "Other than tidying up the structure a little, and some verbose print messages and inline comments, I've also:\n", "- Chose an input image size of (256,256) and increased the U-net's basin 'depth' by 1 extra layer.\n", "- Added in a custom log loss + metric (mean_iou) plot based on [this](https://github.com/deepsense-ai/intel-ai-webinar-neural-networks/blob/master/live_loss_plot.py)\n", "- Add in runtime data augmentation, credits to this [kernel](https://www.kaggle.com/aviwolfson/adding-augmentation-to-keras-u-net-starter) and this [kernel](https://www.kaggle.com/c0conuts/unet-imagedatagenerator-lb-0-336)!\n", "- Add support for multiple GPUs\n", "\n", "TODO:\n", "- Create computational graph visual of the keras model following [this](https://keras.io/visualization/)??"], "metadata": {"_uuid": "dc070efb3529403ae193eb518c646652ddc64f73", "_cell_guid": "00f109e6-9356-4f9a-86fc-92e783ea7e61"}}, {"metadata": {"_uuid": "3a0302dcf95e8b5ba2aff2b600bcebbac86f10ea", "_cell_guid": "5c3d0741-494f-4c00-a1c6-e8307902c3ec", "collapsed": true}, "cell_type": "code", "source": ["# Set number of GPUs\n", "num_gpus = 1   #defaults to 1 if one-GPU or one-CPU. If 4 GPUs, set to 4.\n", "\n", "# Set height (y-axis length) and width (x-axis length) to train model on\n", "img_height, img_width = (256,256)  #Default to (256,266), use (None,None) if you do not want to resize imgs"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "5dabd530b1d74b3275721cc981311f39d6728b18", "_cell_guid": "66a77cab-abe1-44cf-9276-d1b32187cea5", "collapsed": true}, "cell_type": "code", "source": ["# Import all the necessary libraries\n", "import os\n", "import datetime\n", "import glob\n", "import random\n", "import sys\n", "\n", "import matplotlib.pyplot as plt\n", "import skimage.io                                     #Used for imshow function\n", "import skimage.transform                              #Used for resize function\n", "from skimage.morphology import label                  #Used for Run-Length-Encoding RLE to create final submission\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import keras\n", "from keras.layers import Input, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, Conv2DTranspose\n", "from keras.layers import AveragePooling2D, MaxPooling2D, Dropout, GlobalMaxPooling2D, GlobalAveragePooling2D, Lambda\n", "from keras.layers.advanced_activations import LeakyReLU\n", "from keras.models import load_model, Model\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras.layers.merge import add, concatenate\n", "from keras.callbacks import EarlyStopping, ModelCheckpoint\n", "from keras.utils import multi_gpu_model, plot_model\n", "from keras import backend as K\n", "import tensorflow as tf\n", "import sklearn\n", "from sklearn.model_selection import train_test_split\n", "\n", "\n", "print('Python       :', sys.version.split('\\n')[0])\n", "print('Numpy        :', np.__version__)\n", "print('Skimage      :', skimage.__version__)\n", "print('Scikit-learn :', sklearn.__version__)\n", "print('Keras        :', keras.__version__)\n", "print('Tensorflow   :', tf.__version__)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "1bdba649abfdf285b68181bc05be11088ec2e7dd", "_cell_guid": "5d95393a-710e-4240-9d07-148bc6c2180f", "collapsed": true}, "cell_type": "code", "source": ["# Set seed values\n", "seed = 42\n", "random.seed = seed\n", "np.random.seed(seed=seed)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "5c645d5dc179f8aa640021f4a2306cb65873c4b6", "_cell_guid": "1083fdfe-2505-412f-8291-89b6307e2e2d", "collapsed": true}, "cell_type": "code", "source": ["# Have a look at our data folder\n", "topDir = '/kaggle' #defaults to '/kaggle' in kaggle kernels, different if on own system e.g. '/home/user/kaggle/dsbowl'\n", "os.chdir(topDir)    #changes our python working directory to the top directory of our kaggle files\n", "print(os.listdir(os.path.join(topDir, 'input')))  #see what's in the input folder (where data is in)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "5cbd846af776a95d94cf8d3499ee9025b4f5c27f", "_cell_guid": "6793f497-21e6-4618-afd9-7797740dfd28", "collapsed": true}, "cell_type": "code", "source": ["train_path = os.path.join(topDir, 'input/stage1_train')  #path to training data file/folder\n", "test_path = os.path.join(topDir, 'input/stage1_test')   #path to test data file/folder"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["# Part 1 - Data Input"], "metadata": {"_uuid": "a1951e5b2626c7460dda35724fc5503092b89cf9", "_cell_guid": "23b9e37c-8250-439c-b50b-3a4ddc652c3a"}}, {"metadata": {"_uuid": "c8a8f5c908b1d17baa979a45c440920ff9ccf04e", "_cell_guid": "d11df0ce-1602-4c9a-9cdf-fdc3f1226101", "collapsed": true}, "cell_type": "code", "source": ["%%time\n", "# Get training data\n", "def get_X_data(path, output_shape=(None, None)):\n", "    '''\n", "    Loads images from path/{id}/images/{id}.png into a numpy array\n", "    '''\n", "    img_paths = ['{0}/{1}/images/{1}.png'.format(path, id) for id in os.listdir(path)]\n", "    X_data = np.array([skimage.transform.resize(skimage.io.imread(path)[:,:,:3], output_shape=output_shape, mode='constant', preserve_range=True) for path in img_paths], dtype=np.uint8)  #take only 3 channels/bands\n", "    \n", "    return X_data\n", "X_train = get_X_data(train_path, output_shape=(img_height,img_width))\n", "print(X_train.shape, X_train.dtype)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "847cbe8f81fa52cf153be18d7cad215bc88bbe9d", "_cell_guid": "9e92d149-8dfd-40f0-b3c1-bbce47f8d561", "collapsed": true}, "cell_type": "code", "source": ["%%time\n", "# Get training data labels\n", "def get_Y_data(path, output_shape=(None, None)):\n", "    '''\n", "    Loads and concatenates images from path/{id}/masks/{id}.png into a numpy array\n", "    '''\n", "    img_paths = [glob.glob('{0}/{1}/masks/*.png'.format(path, id)) for id in os.listdir(path)]\n", "    \n", "    Y_data = []\n", "    for i, img_masks in enumerate(img_paths):  #loop through each individual nuclei for an image and combine them together\n", "        masks = skimage.io.imread_collection(img_masks).concatenate()  #masks.shape = (num_masks, img_height, img_width)\n", "        mask = np.max(masks, axis=0)                                   #mask.shape = (img_height, img_width)\n", "        mask = skimage.transform.resize(mask, output_shape=output_shape+(1,), mode='constant', preserve_range=True)  #need to add an extra dimension so mask.shape = (img_height, img_width, 1)\n", "        Y_data.append(mask)\n", "    Y_data = np.array(Y_data, dtype=np.bool)\n", "    \n", "    return Y_data\n", "Y_train = get_Y_data(train_path, output_shape=(img_height,img_width))\n", "print(Y_train.shape, Y_train.dtype)"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["## Visualize masks on the training data"], "metadata": {"_uuid": "7cf01d39c5565b48eaba6a2be42518ca7ff02341", "_cell_guid": "1ee80cd2-03ad-4a1b-998e-b4b9600d09d6"}}, {"metadata": {"_uuid": "a2e8483a7b4c4bb5324340ac04434e395f84c793", "_cell_guid": "6fca421e-4d4f-421f-8118-7eb6e4a5cf43", "collapsed": true}, "cell_type": "code", "source": ["id = 64\n", "print(X_train[id].shape)\n", "skimage.io.imshow(X_train[id])\n", "plt.show()\n", "skimage.io.imshow(Y_train[id][:,:,0])\n", "plt.show()"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["# Part 2 - Build model"], "metadata": {"_uuid": "0a50c27113c940bd20e527edc422143ba4f8a9e1", "_cell_guid": "7129b20c-bdc2-43a5-8764-a762e2bf79b2"}}, {"metadata": {"_uuid": "6f9a58dd35b6aa0d43b4bd4dd89b25f52a4ab74f", "_cell_guid": "1d4cdf5b-be18-4abf-86ef-d76e8294474c", "collapsed": true}, "cell_type": "code", "source": ["# Design our model architecture here\n", "def keras_model(img_width=256, img_height=256):\n", "    '''\n", "    Modified from https://keunwoochoi.wordpress.com/2017/10/11/u-net-on-keras-2-0/\n", "    '''\n", "    n_ch_exps = [4, 5, 6, 7, 8, 9]   #the n-th deep channel's exponent i.e. 2**n 16,32,64,128,256\n", "    k_size = (3, 3)                  #size of filter kernel\n", "    k_init = 'he_normal'             #kernel initializer\n", "\n", "    if K.image_data_format() == 'channels_first':\n", "        ch_axis = 1\n", "        input_shape = (3, img_width, img_height)\n", "    elif K.image_data_format() == 'channels_last':\n", "        ch_axis = 3\n", "        input_shape = (img_width, img_height, 3)\n", "\n", "    inp = Input(shape=input_shape)\n", "    encodeds = []\n", "\n", "    # encoder\n", "    enc = inp\n", "    print(n_ch_exps)\n", "    for l_idx, n_ch in enumerate(n_ch_exps):\n", "        enc = Conv2D(filters=2**n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(enc)\n", "        enc = Dropout(0.1*l_idx,)(enc)\n", "        enc = Conv2D(filters=2**n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(enc)\n", "        encodeds.append(enc)\n", "        #print(l_idx, enc)\n", "        if n_ch < n_ch_exps[-1]:  #do not run max pooling on the last encoding/downsampling step\n", "            enc = MaxPooling2D(pool_size=(2,2))(enc)\n", "    \n", "    # decoder\n", "    dec = enc\n", "    print(n_ch_exps[::-1][1:])\n", "    decoder_n_chs = n_ch_exps[::-1][1:]\n", "    for l_idx, n_ch in enumerate(decoder_n_chs):\n", "        l_idx_rev = len(n_ch_exps) - l_idx - 2  #\n", "        dec = Conv2DTranspose(filters=2**n_ch, kernel_size=k_size, strides=(2,2), activation='relu', padding='same', kernel_initializer=k_init)(dec)\n", "        dec = concatenate([dec, encodeds[l_idx_rev]], axis=ch_axis)\n", "        dec = Conv2D(filters=2**n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(dec)\n", "        dec = Dropout(0.1*l_idx)(dec)\n", "        dec = Conv2D(filters=2**n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(dec)\n", "\n", "    outp = Conv2DTranspose(filters=1, kernel_size=k_size, activation='sigmoid', padding='same', kernel_initializer='glorot_normal')(dec)\n", "\n", "    model = Model(inputs=[inp], outputs=[outp])\n", "    \n", "    return model"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "07663a1d6c70143c02c78cdacaeb58f21e835dc0", "_cell_guid": "5017fb68-fe0a-4a65-b9b4-1f1f21912869", "collapsed": true}, "cell_type": "code", "source": ["# Custom IoU metric\n", "def mean_iou(y_true, y_pred):\n", "    prec = []\n", "    for t in np.arange(0.5, 1.0, 0.05):\n", "        y_pred_ = tf.to_int32(y_pred > t)\n", "        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)\n", "        K.get_session().run(tf.local_variables_initializer())\n", "        with tf.control_dependencies([up_opt]):\n", "            score = tf.identity(score)\n", "        prec.append(score)\n", "    return K.mean(K.stack(prec), axis=0)\n", "\n", "# Custom loss function\n", "def dice_coef(y_true, y_pred):\n", "    smooth = 1.\n", "    y_true_f = K.flatten(y_true)\n", "    y_pred_f = K.flatten(y_pred)\n", "    intersection = K.sum(y_true_f * y_pred_f)\n", "    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n", "\n", "def bce_dice_loss(y_true, y_pred):\n", "    return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred) - dice_coef(y_true, y_pred)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "796008f7760b64d768b201ae135e06ac9ce60007", "_cell_guid": "30f3c231-d347-4d44-809a-ba5aa4d2944b", "scrolled": false, "collapsed": true}, "cell_type": "code", "source": ["# Set some model compile parameters\n", "optimizer = 'adam'\n", "loss      = bce_dice_loss\n", "metrics   = [mean_iou]\n", "\n", "# Compile our model\n", "model = keras_model(img_width=img_width, img_height=img_height)\n", "model.summary()\n", "\n", "# For more GPUs\n", "if num_gpus > 1:\n", "    model = multi_gpu_model(model, gpus=num_gpus)\n", "\n", "model.compile(optimizer=optimizer, loss=loss, metrics=metrics)"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["# Part 3 - Run model"], "metadata": {"_uuid": "3ef3e0dfd1e2e22c43c9a880a7e9e77473929ab7", "_cell_guid": "05e830e6-c323-4c39-a707-e52a1c17957b"}}, {"metadata": {"_uuid": "4e95cd22e05505a20deafbf57839ce5e06f82783", "_cell_guid": "c3b00a59-ca4f-4ed8-b38d-c0c007f227c0", "collapsed": true}, "cell_type": "code", "source": ["# Runtime data augmentation\n", "def get_train_test_augmented(X_data=X_train, Y_data=Y_train, validation_split=0.25, batch_size=32, seed=seed):\n", "    X_train, X_test, Y_train, Y_test = train_test_split(X_data,\n", "                                                        Y_data,\n", "                                                        train_size=1-validation_split,\n", "                                                        test_size=validation_split,\n", "                                                        random_state=seed)\n", "    \n", "    # Image data generator distortion options\n", "    data_gen_args = dict(rotation_range=45.,\n", "                         width_shift_range=0.1,\n", "                         height_shift_range=0.1,\n", "                         shear_range=0.2,\n", "                         zoom_range=0.2,\n", "                         horizontal_flip=True,\n", "                         vertical_flip=True,\n", "                         fill_mode='reflect')  #use 'constant'??\n", "\n", "\n", "    # Train data, provide the same seed and keyword arguments to the fit and flow methods\n", "    X_datagen = ImageDataGenerator(**data_gen_args)\n", "    Y_datagen = ImageDataGenerator(**data_gen_args)\n", "    X_datagen.fit(X_train, augment=True, seed=seed)\n", "    Y_datagen.fit(Y_train, augment=True, seed=seed)\n", "    X_train_augmented = X_datagen.flow(X_train, batch_size=batch_size, shuffle=True, seed=seed)\n", "    Y_train_augmented = Y_datagen.flow(Y_train, batch_size=batch_size, shuffle=True, seed=seed)\n", "     \n", "    \n", "    # Test data, no data augmentation, but we create a generator anyway\n", "    X_datagen_val = ImageDataGenerator()\n", "    Y_datagen_val = ImageDataGenerator()\n", "    X_datagen_val.fit(X_test, augment=True, seed=seed)\n", "    Y_datagen_val.fit(Y_test, augment=True, seed=seed)\n", "    X_test_augmented = X_datagen_val.flow(X_test, batch_size=batch_size, shuffle=True, seed=seed)\n", "    Y_test_augmented = Y_datagen_val.flow(Y_test, batch_size=batch_size, shuffle=True, seed=seed)\n", "    \n", "    \n", "    # combine generators into one which yields image and masks\n", "    train_generator = zip(X_train_augmented, Y_train_augmented)\n", "    test_generator = zip(X_test_augmented, Y_test_augmented)\n", "    \n", "    return train_generator, test_generator"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "7a0fbed34c6b95d6b2abd081b7838f9194af0c95", "_cell_guid": "d78cd8da-0905-46e4-b969-37096272e00a", "collapsed": true}, "cell_type": "code", "source": ["# Runtime custom callbacks\n", "#%% https://github.com/deepsense-ai/intel-ai-webinar-neural-networks/blob/master/live_loss_plot.py\n", "# Fixed code to enable non-flat loss plots on keras model.fit_generator()\n", "import matplotlib.pyplot as plt\n", "from keras.callbacks import Callback\n", "from IPython.display import clear_output\n", "#from matplotlib.ticker import FormatStrFormatter\n", "\n", "def translate_metric(x):\n", "    translations = {'acc': \"Accuracy\", 'loss': \"Log-loss (cost function)\"}\n", "    if x in translations:\n", "        return translations[x]\n", "    else:\n", "        return x\n", "\n", "class PlotLosses(Callback):\n", "    def __init__(self, figsize=None):\n", "        super(PlotLosses, self).__init__()\n", "        self.figsize = figsize\n", "\n", "    def on_train_begin(self, logs={}):\n", "\n", "        self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]\n", "        self.logs = []\n", "\n", "    def on_epoch_end(self, epoch, logs={}):\n", "        self.logs.append(logs.copy())\n", "\n", "        clear_output(wait=True)\n", "        plt.figure(figsize=self.figsize)\n", "        \n", "        for metric_id, metric in enumerate(self.base_metrics):\n", "            plt.subplot(1, len(self.base_metrics), metric_id + 1)\n", "            \n", "            plt.plot(range(1, len(self.logs) + 1),\n", "                     [log[metric] for log in self.logs],\n", "                     label=\"training\")\n", "            if self.params['do_validation']:\n", "                plt.plot(range(1, len(self.logs) + 1),\n", "                         [log['val_' + metric] for log in self.logs],\n", "                         label=\"validation\")\n", "            plt.title(translate_metric(metric))\n", "            plt.xlabel('epoch')\n", "            plt.legend(loc='center left')\n", "        \n", "        plt.tight_layout()\n", "        plt.show();\n", "\n", "plot_losses = PlotLosses(figsize=(16, 4))"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "b7b93a826a4b4337af83cf9999524035e7470601", "_cell_guid": "cec5a1ff-c983-426d-a0da-581f25966860", "scrolled": false, "collapsed": true}, "cell_type": "code", "source": ["# Finally train the model!!\n", "batch_size = 16\n", "\n", "train_generator, test_generator = get_train_test_augmented(X_data=X_train, Y_data=Y_train, validation_split=0.11, batch_size=batch_size)\n", "model.fit_generator(train_generator, validation_data=test_generator, validation_steps=batch_size/2, steps_per_epoch=len(X_train)/(batch_size*2), epochs=5, callbacks=[plot_losses])"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "3996b22c52d1a2b9e5a0759b6a2c89f1ec492c8a", "_cell_guid": "ead14275-6038-4a00-998d-78a376755e81", "scrolled": false, "collapsed": true}, "cell_type": "code", "source": ["# Save the model weights to a hdf5 file\n", "if num_gpus > 1:\n", "    #Refer to https://stackoverflow.com/questions/41342098/keras-load-checkpoint-weights-hdf5-generated-by-multiple-gpus\n", "    #model.summary()\n", "    model_out = model.layers[-2]  #get second last layer in multi_gpu_model i.e. model.get_layer('model_1')\n", "else:\n", "    model_out = model\n", "model_out.save_weights(filepath=topDir+\"/working/model-weights.hdf5\")"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["# Part 4 - Evaluate output"], "metadata": {"_uuid": "d8b70e56c01002b673e354223a00df0c17b8cb83", "_cell_guid": "2cf8de83-1486-444d-9200-e1c2784b25cc"}}, {"metadata": {"_uuid": "0503cb914b189ff235490c8647cbb3b12d9ef41d", "_cell_guid": "9afda42c-9664-437d-9c95-7dfc134a0a9c", "collapsed": true}, "cell_type": "code", "source": ["# Reload the model\n", "model_loaded = keras_model(img_width=img_width, img_height=img_height)\n", "model_loaded.load_weights(topDir+\"/working/model-weights.hdf5\")"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "23a71c7299667761f811e658ad74b08a5fbbb8d9", "_cell_guid": "bd5ccd5b-72c5-4fc0-a327-fb07d54d636f", "collapsed": true}, "cell_type": "code", "source": ["# Get test data\n", "X_test = get_X_data(test_path, output_shape=(img_height,img_width))"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "9179437a04bf2e63fc9f54d118bdd8840f30636a", "_cell_guid": "a78cd1cd-f47f-482a-bc7b-b1203beba9ac", "collapsed": true}, "cell_type": "code", "source": ["# Use model to predict test labels\n", "Y_hat = model_loaded.predict(X_test, verbose=1)\n", "Y_hat.shape"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["## Visualize predictions on the test data"], "metadata": {"_uuid": "3c033ee95aad28db6952110d8e23765fbb6de25a", "_cell_guid": "0ea30110-a9c6-4291-a41a-fd97f3960a25"}}, {"metadata": {"_uuid": "2c41b65dd7eab34ccff6b99f63c48551526d3e88", "_cell_guid": "8acf23c2-d28b-4bfc-8b0b-f054bbcba93a", "collapsed": true}, "cell_type": "code", "source": ["id = 32\n", "print(X_test[id].shape)\n", "skimage.io.imshow(X_test[id])\n", "plt.show()\n", "skimage.io.imshow(Y_hat[id][:,:,0])\n", "plt.show()"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["# Part 5 - Submit results"], "metadata": {"_uuid": "25f599e017681e4a269bfc2dc6640bfb4532e2ff", "_cell_guid": "88938860-0dc9-4b99-bc53-292a0758a90a"}}, {"metadata": {"_uuid": "f3dc5bd6c2fda3943072532b551d52fdf0be0df3", "_cell_guid": "5fa8be0e-ae9c-43ee-9c4c-9dd81e750595", "collapsed": true}, "cell_type": "code", "source": ["# Upsample Y_hat back to the original X_test size (height and width)\n", "Y_hat_upsampled = []\n", "for i, test_id in enumerate(os.listdir(test_path)):  #loop through test_ids in the test_path\n", "    img = skimage.io.imread('{0}/{1}/images/{1}.png'.format(test_path, test_id))  #read original test image directly from path\n", "    img_upscaled = skimage.transform.resize(Y_hat[i], (img.shape[0], img.shape[1]), mode='constant', preserve_range=True)  #upscale Y_hat image according to original test image\n", "    Y_hat_upsampled.append(img_upscaled)   #append upscaled image to Y_hat_upsampled\n", "len(Y_hat_upsampled)"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "source": ["## Visualize upscaled predictions on the test data"], "metadata": {"_uuid": "7df12e36f3454c6e608f3dee1f5f806d1ef88602", "_cell_guid": "4abe8dc6-7901-4992-b2fc-f9637d286672"}}, {"metadata": {"_uuid": "a9d5ceecefb5c559358a91af0b8907e8d7102675", "_cell_guid": "f1279fdd-2212-465f-a2b1-bdde778ca174", "collapsed": true}, "cell_type": "code", "source": ["id = 32\n", "print(Y_hat_upsampled[id].shape)\n", "skimage.io.imshow(Y_hat_upsampled[id][:,:,0])"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "382656049394a64cf52732415ff727ba3f43357d", "_cell_guid": "d17b2845-5fb7-426e-bef1-2728cd576dc4", "collapsed": true}, "cell_type": "code", "source": ["# Run-length encoding stolen from https://www.kaggle.com/rakhlin/fast-run-length-encoding-python\n", "def rle_encoding(x):\n", "    dots = np.where(x.T.flatten() == 1)[0]\n", "    run_lengths = []\n", "    prev = -2\n", "    for b in dots:\n", "        if (b>prev+1): run_lengths.extend((b + 1, 0))\n", "        run_lengths[-1] += 1\n", "        prev = b\n", "    return run_lengths\n", "\n", "def prob_to_rles(x, cutoff=0.5):\n", "    lab_img = label(x > cutoff)\n", "    for i in range(1, lab_img.max() + 1):\n", "        yield rle_encoding(lab_img == i)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "df6b5e0f4af79cc12d1df0b102ffbb896e75aa92", "_cell_guid": "261083fd-2623-4c31-9914-357beaeceace", "collapsed": true}, "cell_type": "code", "source": ["# Apply Run-Length Encoding on our Y_hat_upscaled\n", "new_test_ids = []\n", "rles = []\n", "for n, id_ in enumerate(os.listdir(test_path)):\n", "    rle = list(prob_to_rles(Y_hat_upsampled[n]))\n", "    rles.extend(rle)\n", "    new_test_ids.extend([id_] * len(rle))\n", "len(new_test_ids)  #note that for each test_image, we can have multiple entries of encoded pixels"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "d5d8b56f676a732d83d83b979ea30d937700d0cc", "_cell_guid": "5acd063c-ddb8-47ae-96c6-14693cd90d6a", "collapsed": true}, "cell_type": "code", "source": ["# Create submission DataFrame\n", "sub = pd.DataFrame()\n", "sub['ImageId'] = new_test_ids\n", "sub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\n", "timestamp = datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\n", "print('Submission output to: sub-{}.csv'.format(timestamp))\n", "sub.to_csv(topDir+\"/working/sub-{}.csv\".format(timestamp), index=False)"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "59070d50687f10fabc250fb73c9d0fed355f67a1", "_cell_guid": "4ece2ece-5d06-4467-bb2f-fc7eb59badc3", "scrolled": true, "collapsed": true}, "cell_type": "code", "source": ["# Have a look at our submission pandas dataframe\n", "sub.head()"], "execution_count": null, "outputs": []}, {"metadata": {"_uuid": "64201589be0e22ab2bfda9bfddcbaf1302588b3c", "_cell_guid": "6d223b68-24fe-4707-80a5-caec183853f3", "collapsed": true}, "cell_type": "code", "source": [], "execution_count": null, "outputs": []}], "nbformat": 4, "metadata": {"kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}, "language_info": {"pygments_lexer": "ipython3", "version": "3.6.4", "codemirror_mode": {"name": "ipython", "version": 3}, "name": "python", "file_extension": ".py", "nbconvert_exporter": "python", "mimetype": "text/x-python"}}, "nbformat_minor": 1}