{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# U-NET for Nuclei Segmentation \n# U-Net 细胞核分割\n\nhttps://www.kaggle.com/c/data-science-bowl-2018","metadata":{}},{"cell_type":"code","source":"# Set number of GPUs\nnum_gpus = 1   #defaults to 1 if one-GPU or one-CPU. If 4 GPUs, set to 4.\n\n# Set height (y-axis length) and width (x-axis length) to train model on\nimg_height, img_width = (256,256)  #Default to (256,266), use (None,None) if you do not want to resize imgs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tensorflow==1.15\nimport tensorflow as tf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install keras==2.2.4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import all the necessary libraries\nimport os\nimport datetime\nimport glob\nimport random\nimport sys\n\nimport matplotlib.pyplot as plt\nimport skimage.io                                     #Used for imshow function\nimport skimage.transform                              #Used for resize function\nfrom skimage.morphology import label                  #Used for Run-Length-Encoding RLE to create final submission\nimport cv2\nimport numpy as np\nimport pandas as pd\n\nimport keras\nfrom keras.layers import Input, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, Conv2DTranspose\nfrom keras.layers import AveragePooling2D, MaxPooling2D, Dropout, GlobalMaxPooling2D, GlobalAveragePooling2D, Lambda\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.models import load_model, Model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers.merge import add, concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.utils import plot_model\n# from keras.utils import multi_gpu_model\nfrom keras import backend as K\nimport tensorflow as tf\nimport sklearn\nfrom sklearn.model_selection import train_test_split\n\nprint('Python       :', sys.version.split('\\n')[0])\nprint('Numpy        :', np.__version__)\nprint('Skimage      :', skimage.__version__)\nprint('Scikit-learn :', sklearn.__version__)\nprint('Keras        :', keras.__version__)\nprint('Tensorflow   :', tf.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set seed values\nseed = 42\nrandom.seed = seed\nnp.random.seed(seed=seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set seed values\nseed = 42\nrandom.seed = seed\nnp.random.seed(seed=seed)","metadata":{}},{"cell_type":"code","source":"!cd ../\n!ls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Have a look at our data folder\n# topDir = '/home/' #defaults to '/kaggle' in kaggle kernels, different if on own system e.g. '/home/user/kaggle/dsbowl'\n# os.chdir(topDir)    #changes our python working directory to the top directory of our kaggle files\n# print(os.listdir(os.path.join(topDir, 'input')))  #see what's in the input folder (where data is in)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topDir='../input/datascience2018/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = os.path.join(topDir, 'input/stage1_train')  #path to training data file/folder\ntest_path = os.path.join(topDir, 'input/stage1_test')   #path to test data file/folder\ntest_final_path= os.path.join(topDir, 'input/stage2_test_final') ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 1 - 加载数据","metadata":{}},{"cell_type":"code","source":"#%%time\n# Get training data\ndef get_X_data(path, output_shape=(None, None)):\n    '''\n    Loads images from path/{id}/images/{id}.png into a numpy array\n    '''\n    img_paths = ['{0}/{1}/images/{1}.png'.format(path, id) for id in os.listdir(path)]\n    X_data = np.array([skimage.transform.resize(skimage.io.imread(path)[:, :, :3], output_shape=output_shape, mode='constant', preserve_range=True) for path in img_paths],dtype=np.uint8)  # take only 3 channels/bands\n\n    return X_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = get_X_data(train_path, output_shape=(img_height, img_width))\nprint(X_train.shape, X_train.dtype)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\n# Get training data labels\ndef get_Y_data(path, output_shape=(None, None)):\n    '''\n    Loads and concatenates images from path/{id}/masks/{id}.png into a numpy array\n    '''\n    img_paths = [glob.glob('{0}/{1}/masks/*.png'.format(path, id)) for id in os.listdir(path)]\n\n    Y_data = []\n    for i, img_masks in enumerate(img_paths):  # loop through each individual nuclei for an image and combine them together\n        masks = skimage.io.imread_collection(img_masks).concatenate()  # masks.shape = (num_masks, img_height, img_width)\n        mask = np.max(masks, axis=0)  # mask.shape = (img_height, img_width)\n        mask = skimage.transform.resize(mask, output_shape=output_shape + (1,), mode='constant', preserve_range=True)  # need to add an extra dimension so mask.shape = (img_height, img_width, 1)\n        Y_data.append(mask)\n    Y_data = np.array(Y_data, dtype=np.bool)\n\n    return Y_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = get_Y_data(train_path, output_shape=(img_height, img_width))\nprint(Y_train.shape, Y_train.dtype)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Illustrate the train images and masks\nplt.figure(figsize=(20,16))\nx, y = 12,4\nfor i in range(y):\n    for j in range(x):\n        # train image\n        plt.subplot(y*2, x, i*2*x+j+1)\n        pos = i*120 + j*10\n        plt.imshow(X_train[pos])\n        plt.title('Image #{}'.format(pos))\n        plt.axis('off')\n        plt.subplot(y*2, x, (i*2+1)*x+j+1)\n        plt.imshow(np.squeeze(Y_train[pos]))\n        plt.title('Mask #{}'.format(pos))\n        plt.axis('off')\n#plt.subplots_adjust(wspace=0.1, hspace=0.1)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 2 - 构建网络模型","metadata":{}},{"cell_type":"code","source":"# Design our model architecture here\ndef keras_model(img_width=256, img_height=256):\n    '''\n    Modified from https://keunwoochoi.wordpress.com/2017/10/11/u-net-on-keras-2-0/\n    '''\n    n_ch_exps = [4, 5, 6, 7, 8, 9]  # the n-th deep channel's exponent i.e. 2**n 16,32,64,128,256\n    k_size = (3, 3)  # size of filter kernel\n    k_init = 'he_normal'  # kernel initializer\n\n    if K.image_data_format() == 'channels_first':\n        ch_axis = 1\n        input_shape = (3, img_width, img_height)\n    elif K.image_data_format() == 'channels_last':\n        ch_axis = 3\n        input_shape = (img_width, img_height, 3)\n\n    inp = Input(shape=input_shape)\n    encodeds = []\n\n    # encoder\n    enc = inp\n    print(n_ch_exps)\n    for l_idx, n_ch in enumerate(n_ch_exps):\n        enc = Conv2D(filters=2 ** n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(enc)\n        enc = Dropout(0.1 * l_idx, )(enc)\n        enc = Conv2D(filters=2 ** n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(enc)\n        encodeds.append(enc)\n        # print(l_idx, enc)\n        if n_ch < n_ch_exps[-1]:  # do not run max pooling on the last encoding/downsampling step\n            enc = MaxPooling2D(pool_size=(2, 2))(enc)\n\n    # decoder\n    dec = enc\n    print(n_ch_exps[::-1][1:])\n    decoder_n_chs = n_ch_exps[::-1][1:]\n    for l_idx, n_ch in enumerate(decoder_n_chs):\n        l_idx_rev = len(n_ch_exps) - l_idx - 2  #\n        dec = Conv2DTranspose(filters=2 ** n_ch, kernel_size=k_size, strides=(2, 2), activation='relu', padding='same', kernel_initializer=k_init)(dec)\n        dec = concatenate([dec, encodeds[l_idx_rev]], axis=ch_axis)\n        dec = Conv2D(filters=2 ** n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(dec)\n        dec = Dropout(0.1 * l_idx)(dec)\n        dec = Conv2D(filters=2 ** n_ch, kernel_size=k_size, activation='relu', padding='same', kernel_initializer=k_init)(dec)\n\n    outp = Conv2DTranspose(filters=1, kernel_size=k_size, activation='sigmoid', padding='same', kernel_initializer='glorot_normal')(dec)\n\n    model = Model(inputs=[inp], outputs=[outp])\n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom IoU metric\ndef mean_iou(y_true, y_pred):\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        y_pred_ = tf.to_int32(y_pred > t)\n        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)\n        K.get_session().run(tf.local_variables_initializer())\n        with tf.control_dependencies([up_opt]):\n            score = tf.identity(score)\n        prec.append(score)\n    return K.mean(K.stack(prec), axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom loss function\ndef dice_coef(y_true, y_pred):\n    smooth = 1.\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bce_dice_loss(y_true, y_pred):\n    return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred) - dice_coef(y_true, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set some model compile parameters\noptimizer = 'adam'\nloss      = bce_dice_loss\nmetrics   = [mean_iou]\n\n# Compile our model\nmodel = keras_model(img_width=img_width, img_height=img_height)\nmodel.summary()\n\n# For more GPUs\nif num_gpus > 1:\n    model = multi_gpu_model(model, gpus=num_gpus)\n\nmodel.compile(optimizer=optimizer, loss=loss, metrics=metrics)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 3 - 训练网络模型","metadata":{}},{"cell_type":"code","source":"# Runtime data augmentation\ndef get_train_test_augmented(X_data=X_train, Y_data=Y_train, validation_split=0.1, batch_size=32, seed=seed):\n    X_train, X_test, Y_train, Y_test = train_test_split(X_data,\n                                                        Y_data,\n                                                        train_size=1 - validation_split,\n                                                        test_size=validation_split,\n                                                        random_state=seed)\n    # Image data generator distortion options\n    data_gen_args = dict(rotation_range=45.,\n                         width_shift_range=0.1,\n                         height_shift_range=0.1,\n                         shear_range=0.2,\n                         zoom_range=0.2,\n                         horizontal_flip=True,\n                         vertical_flip=True,\n                         fill_mode='reflect')  # use 'constant'??\n\n    # Train data, provide the same seed and keyword arguments to the fit and flow methods\n    X_datagen = ImageDataGenerator(**data_gen_args)\n    Y_datagen = ImageDataGenerator(**data_gen_args)\n    X_datagen.fit(X_train, augment=True, seed=seed)\n    Y_datagen.fit(Y_train, augment=True, seed=seed)\n    X_train_augmented = X_datagen.flow(X_train, batch_size=batch_size, shuffle=True, seed=seed)\n    Y_train_augmented = Y_datagen.flow(Y_train, batch_size=batch_size, shuffle=True, seed=seed)\n\n    # Test data, no data augmentation, but we create a generator anyway\n    X_datagen_val = ImageDataGenerator()\n    Y_datagen_val = ImageDataGenerator()\n    X_datagen_val.fit(X_test, augment=True, seed=seed)\n    Y_datagen_val.fit(Y_test, augment=True, seed=seed)\n    X_test_augmented = X_datagen_val.flow(X_test, batch_size=batch_size, shuffle=True, seed=seed)\n    Y_test_augmented = Y_datagen_val.flow(Y_test, batch_size=batch_size, shuffle=True, seed=seed)\n\n    # combine generators into one which yields image and masks\n    train_generator = zip(X_train_augmented, Y_train_augmented)\n    test_generator = zip(X_test_augmented, Y_test_augmented)\n\n    return train_generator, test_generator, X_train, X_test, Y_train, Y_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install livelossplot\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Runtime custom callbacks\n# %% https://github.com/deepsense-ai/intel-ai-webinar-neural-networks/blob/master/live_loss_plot.py\n# Fixed code to enable non-flat loss plots on keras model.fit_generator()\nimport matplotlib.pyplot as plt\nfrom keras.callbacks import Callback\nfrom IPython.display import clear_output\n#from livelossplot.keras import PlotLossesCallback\n# from matplotlib.ticker import FormatStrFormatter\n\ndef translate_metric(x):\n    translations = {'acc': \"Accuracy\", 'loss': \"Log-loss (cost function)\"}\n    if x in translations:\n        return translations[x]\n    else:\n        return x\n\n\nclass PlotLosses(Callback):\n    def __init__(self, figsize=None):\n        super(PlotLosses, self).__init__()\n        self.figsize = figsize\n\n    def on_train_begin(self, logs={}):\n\n        self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]\n        self.logs = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        self.logs.append(logs.copy())\n\n        clear_output(wait=True)\n        plt.figure(figsize=self.figsize)\n\n        for metric_id, metric in enumerate(self.base_metrics):\n            plt.subplot(1, len(self.base_metrics), metric_id + 1)\n\n            plt.plot(range(1, len(self.logs) + 1),\n                     [log[metric] for log in self.logs],\n                     label=\"training\")\n            if self.params['do_validation']:\n                plt.plot(range(1, len(self.logs) + 1),\n                         [log['val_' + metric] for log in self.logs],\n                         label=\"validation\")\n            plt.title(translate_metric(metric))\n            plt.xlabel('epoch')\n            plt.legend(loc='center left')\n\n        plt.tight_layout()\n        plt.show();\n\nplot_losses = PlotLosses(figsize=(16, 4))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finally train the model!!\nbatch_size = 16\n\ntrain_generator, test_generator, X_train, X_val, Y_train, Y_val = get_train_test_augmented(X_data=X_train, Y_data=Y_train, validation_split=0.1, batch_size=batch_size)\n# increase epoch on your own machine\nmodel.fit_generator(train_generator, validation_data=test_generator, validation_steps=batch_size/2, steps_per_epoch=len(X_train)/(batch_size*2), epochs=30, callbacks=[plot_losses])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model weights to a hdf5 file\nif num_gpus > 1:\n    #Refer to https://stackoverflow.com/questions/41342098/keras-load-checkpoint-weights-hdf5-generated-by-multiple-gpus\n    #model.summary()\n    model_out = model.layers[-2]  #get second last layer in multi_gpu_model i.e. model.get_layer('model_1')\nelse:\n    model_out = model\n#working_dir = os.path.join(topDir, \"/working/\")\n#if not os.path.isdir(working_dir):\n#    os.mkdir(working_dir)\nmodel_out.save_weights(filepath=\"./model-weights.hdf5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 4 - 性能评估","metadata":{}},{"cell_type":"markdown","source":"### 验证集数据的预测","metadata":{}},{"cell_type":"code","source":"# Reload the model\nmodel_loaded = keras_model(img_width=img_width, img_height=img_height)\nmodel_loaded.load_weights(\"./model-weights.hdf5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on val\npreds_val = model.predict(X_val, verbose=1)\n# Threshold predictions\n#preds_val_t = (preds_val > 0.5).astype(np.uint8)\npreds_val_t = (preds_val > 0.5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define IoU metric as a regular function, to manually check result\ndef cal_iou(A, B):\n    intersection = np.logical_and(A, B)\n    union = np.logical_or(A, B)\n    iou = np.sum(intersection > 0) / np.sum(union > 0)\n    return iou","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calcualte average iou of validation images, the result from tensorflow seems too high.\niou=[]\nfor i in range(len(Y_val)):\n    iou.append(cal_iou(np.squeeze(Y_val[i]), np.squeeze(preds_val_t[i])))\nprint('Average Validate IOU: {}'.format(round(np.mean(iou),2)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.figure(figsize=(20,10.5))\nplt.figure(figsize=(20,16))\nx, y = 16,3\nfor i in range(y):\n    for j in range(x):\n        # train image\n        plt.subplot(y*3, x, i*3*x+j+1)\n        pos = i*x+j\n        plt.imshow(X_val[pos])\n        plt.title('Image #{}\\nIOU {}'.format(pos,round(cal_iou(np.squeeze(Y_val[pos]), np.squeeze(preds_val_t[pos])),2)))\n        plt.axis('off')\n        plt.subplot(y*3, x, (i*3+1)*x+j+1)\n        plt.imshow(np.squeeze(Y_val[pos]))\n        plt.title('Mask')\n        plt.axis('off')\n        plt.subplot(y*3, x, (i*3+2)*x+j+1)\n        plt.imshow(np.squeeze(preds_val_t[pos]))\n        plt.title('Predict')\n        plt.axis('off')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# illustrate the details for some images\nixs = [8,17,18]\nplt.figure(figsize=(16,8))\nx, y =  6, len(ixs)\nfor i, ix in enumerate(ixs):  \n    # original\n    plt.subplot(y,x, i*x+1)\n    plt.imshow(X_val[ix])\n    plt.title('#{} IOU: {}'.format(ix+1,round(cal_iou(np.squeeze(Y_val[ix]), np.squeeze(preds_val_t[ix])),2)))\n    plt.axis('off')\n    # grand true\n    plt.subplot(y,x, i*x+2)\n    plt.imshow(np.squeeze(Y_val[ix]))\n    plt.title('Mask')\n    plt.axis('off')\n    # prediction\n    plt.subplot(y,x, i*x+3)\n    pred = preds_val_t[ix]>0\n    plt.imshow(np.squeeze(pred))\n    plt.title('Prediction')\n    plt.axis('off')\n    # true positive\n    plt.subplot(y,x, i*x+4)\n    fp = np.logical_and(pred, Y_val[ix])  \n    plt.imshow(np.squeeze(fp))\n    plt.title('True Positive')\n    plt.axis('off')   \n    # false positive\n    plt.subplot(y,x, i*x+5)\n    fp = np.logical_and(pred, np.logical_not(Y_val[ix]))  \n    plt.imshow(np.squeeze(fp))\n    plt.title('False Positive') \n    plt.axis('off')       \n    # false negtive\n    plt.subplot(y,x, i*x+6)\n    fn = np.logical_and(np.logical_not(pred), Y_val[ix]) \n    plt.imshow(np.squeeze(fn))\n    plt.title('False Negtive')     \n    plt.axis('off')   \nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get test data\nX_test = get_X_data(test_path, output_shape=(img_height,img_width))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\n# Get training data\ndef get_final_data(path, output_shape=(None, None)):\n    '''\n    Loads images from path/{id}/images/{id}.png into a numpy array\n    '''\n    X_data=[]\n    for id in os.listdir(path):\n        paths='{0}/{1}/images/{1}.png'.format(path, id)\n        \n        if id=='170bc41b2095177cccd3d4c8977c619147580f1d93b4fe9701eddd77736d4ece':\n            x=cv2.imread(paths)\n            X_data.append(skimage.transform.resize(x, output_shape=output_shape, mode='constant', preserve_range=True))\n        else:\n            X_data.append(skimage.transform.resize(skimage.io.imread(paths), output_shape=output_shape, mode='constant', preserve_range=True) ) # take only 3 channels/bands\n\n    return np.array(X_data,dtype=np.uint8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #%%time\n# # Get training data\n# def get_final_data(path, output_shape=(None, None)):\n#     '''\n#     Loads images from path/{id}/images/{id}.png into a numpy array\n#     '''\n#     img_paths = ['{0}/{1}/images/{1}.png'.format(path, id) for id in os.listdir(path)]\n#     X_data = np.array([skimage.transform.resize(skimage.io.imread(path), output_shape=output_shape, mode='constant', preserve_range=True) for path in img_paths],dtype=np.uint8)  # take only 3 channels/bands\n\n#     return X_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_final =get_final_data(test_final_path, output_shape=(img_height,img_width,3))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.shape\nX_test_final.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use model to predict test labels\nY_hat = model_loaded.predict(X_test, verbose=1)\nY_hat.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_final=model_loaded.predict(X_test_final, verbose=1)\nY_final.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 测试集预测","metadata":{}},{"cell_type":"code","source":"id = 32\nprint(X_test[id].shape)\nskimage.io.imshow(X_test[id])\nplt.show()\nskimage.io.imshow(Y_hat[id][:,:,0])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 5 - 提交结果","metadata":{}},{"cell_type":"code","source":"# Upsample Y_hat back to the original X_test size (height and width)\nY_hat_upsampled = []\nfor i, test_id in enumerate(os.listdir(test_path)):  #loop through test_ids in the test_path\n    img = skimage.io.imread('{0}/{1}/images/{1}.png'.format(test_path, test_id))  #read original test image directly from path\n    img_upscaled = skimage.transform.resize(Y_hat[i], (img.shape[0], img.shape[1]), mode='constant', preserve_range=True)  #upscale Y_hat image according to original test image\n    Y_hat_upsampled.append(img_upscaled)   #append upscaled image to Y_hat_upsampled\nlen(Y_hat_upsampled)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_final_upsampled = []\nfor i, final_test_id in enumerate(os.listdir(test_final_path)):  #loop through test_ids in the test_path\n    img = cv2.imread('{0}/{1}/images/{1}.png'.format(test_final_path, final_test_id))  #read original test image directly from path\n    img_upscaled = skimage.transform.resize(Y_final[i], (img.shape[0], img.shape[1]), mode='constant', preserve_range=True)  #upscale Y_final image according to original test image\n    Y_final_upsampled.append(img_upscaled)   #append upscaled image to Y_final_upsampled\nlen(Y_final_upsampled)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize upscaled predictions on the test data","metadata":{}},{"cell_type":"code","source":"id = 32\nprint(Y_hat_upsampled[id].shape)\nskimage.io.imshow(Y_hat_upsampled[id][:,:,0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run-length encoding from https://www.kaggle.com/rakhlin/fast-run-length-encoding-python\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cutoff=0.5):\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        yield rle_encoding(lab_img == i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply Run-Length Encoding on our Y_hat_upscaled\nnew_test_ids = []\nrles = []\nfor n, id_ in enumerate(os.listdir(test_path)):\n    rle = list(prob_to_rles(Y_hat_upsampled[n]))\n    rles.extend(rle)\n    new_test_ids.extend([id_] * len(rle))\nlen(new_test_ids)  #note that for each test_image, we can have multiple entries of encoded pixels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply Run-Length Encoding on our Y_hat_upscaled\nnew_test_final_ids = []\nrles_final = []\nfor n, id_ in enumerate(os.listdir(test_final_path)):\n    rle_final = list(prob_to_rles(Y_final_upsampled[n]))\n    rles_final.extend(rle_final)\n    new_test_final_ids.extend([id_] * len(rle_final))\nlen(new_test_final_ids)  #note that for each test_image, we can have multiple entries of encoded pixels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create submission DataFrame\nsub = pd.DataFrame()\nsub['ImageId'] = new_test_ids\nsub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\ntimestamp = datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\nprint('Submission output to: sub-{}.csv'.format(timestamp))\nsub.to_csv(\"./sub-{}.csv\".format(timestamp), index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create submission DataFrame\nsub1 = pd.DataFrame()\nsub1['ImageId'] = new_test_final_ids\nsub1['EncodedPixels'] = pd.Series(rles_final).apply(lambda x: ' '.join(str(y) for y in x))\ntimestamp = datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\nprint('Submission output to: sub1-{}.csv'.format(timestamp))\nsub1.to_csv(\"./sub1-{}.csv\".format(timestamp), index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at our submission pandas dataframe\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}