{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"version":3,"name":"ipython"},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py","version":"3.6.4","name":"python","mimetype":"text/x-python"}},"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"4a938561-4979-41e2-a3a9-4ce96d7e70d2","_uuid":"f51d558c33b8d1ea902881c6680652a92bd5c26d"},"source":"# Introduction\n\nThis notebook will be a summary of several test done on this project. During training, I had issues to train a stable U-Net \"classic\". In order to stabilize it, I decided to train it using 3 images as inputs with differents pré-processing (I'll call this model <b>U-Net extended</b>). In this Notebook, I'll share with you :\n\n<ul>\n<li>A quick exploration</li>\n<li>The pre-processing for the U-net Extended</li>\n<li>The model</li>\n<li>The post-processing</li>\n<li>The prediction Kaggle</li>\n<li>A bonus</li>\n</ul>\n\nLet's go !"},{"cell_type":"markdown","metadata":{"_cell_guid":"ec8b17c6-0375-4c5d-acf1-7b35807959f0","_uuid":"af64135d7976b4c9dcb780eb88f821ecc27aa0b5"},"source":"## Imports and functions\n\nIn order to simplify the reset of the Kernel, functions and imports are stacked on top"},{"cell_type":"code","metadata":{"_cell_guid":"9284d05e-084c-41fa-9060-c06d9b7b7cd2","_uuid":"c560c51ed86a4eb64c9cc877abbd5e48c5dfe2a1","collapsed":true},"source":"import os\nimport numpy as np\nimport glob\nimport cv2\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport pickle\nimport time\n\nfrom collections import Counter\nfrom shutil import copyfile\nfrom PIL import Image\nfrom IPython import display \nfrom sklearn.model_selection import train_test_split\nfrom skimage.morphology import label\nfrom itertools import groupby\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\n%matplotlib inline","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"36b3e326-7d1e-4de7-9daf-bf88fcd87572","_uuid":"99eb59c78605440a368eca1dde667350e897364b","collapsed":true},"source":"from keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import Callback\nfrom keras.optimizers import Adam, SGD\nfrom keras import regularizers\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"057a2eab-69d5-4e51-bfa2-1789783f904e","_uuid":"296a29ff73a72d62290f402f7557416fc4583c1b"},"source":"Here, we will add all functions required for the notebook. This is just to load them easily when we restart the Kernel."},{"cell_type":"code","metadata":{"_cell_guid":"31ec101f-1ea5-4ceb-9594-71fa7f3fdac5","_uuid":"4ea9d5877758da71fbe119fecce6402aa75fce70","collapsed":true},"source":"def save_obj(obj, name ):\n    with open(name + '.pkl', 'wb') as f:\n        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n        \ndef load_obj(name):\n    with open(name + '.pkl', 'rb') as f:\n        return pickle.load(f)\n\ndef is_back_and_white(img):\n    # detect f image is B&W or color\n    a = np.array_equiv(image[:, :, 0], image[:, :, 1])\n    b = np.array_equiv(image[:, :, 1], image[:, :, 2])\n    return (a and b)\n\ndef boost_contrast(img):\n    # equivalent of 255*MinMaxScaling function\n    mini = img.min()\n    maxi = img.max()\n    return (img-mini)*(255/(maxi-mini))\n\ndef add_seuillage(img, level=100):\n    # this function boost again contrasts and clip it to \"ease\" learning\n    img = (img-level)*(255/(255-(2*level)))\n    img = np.clip(img, 0, 255)\n    return img\n\ndef prepare_image_final(image):\n    # pre-process image\n    if not is_back_and_white(image):\n        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        gray = cv2.bitwise_not(gray)\n        gray = boost_contrast(gray)\n        gray = add_seuillage(gray, level=50)\n    else:\n        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        if gray.mean() > 150:\n            gray = cv2.bitwise_not(gray)\n        gray = boost_contrast(gray)\n        gray = add_seuillage(gray, level=0)\n    return gray\n\ndef resize_to_original(img, target_size):\n    current_height, current_width = img.shape\n    target_height, target_width = target_size\n    if current_height >= target_height and current_width >= target_width:\n        # we are in case where we expanded with black\n        return img[:target_height, :target_width]\n    else:\n        scale = max(target_height, target_width) / 512\n        scaled_image = cv2.resize(img, None, fx=scale, fy=scale, interpolation = cv2.INTER_NEAREST)\n        return scaled_image[:target_height, :target_width]\n    return img\n\ndef compute_IoU(pred, mask, trigger=0.999):\n    pred = (pred>trigger)\n    mask = (mask>0.9)\n    union = np.logical_or(pred, mask)\n    intersection = np.logical_and(pred, mask)\n    return intersection.sum()/union.sum()\n\ndef img_to_rle(img):\n    # I prefer groupby vs loop and Flatten('F') avoid the transpose\n    result = []\n    X = img.flatten('F').tolist()\n    current_index = 1\n    for k, g in groupby(X):\n        l = len(list(g))\n        if k > 0.5:\n            result.append(current_index)\n            result.append(l)\n        current_index += l\n    return result\n\n\"\"\"\nFrom Kaggle but I prefer groupby vs loop\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\"\"\"\n\ndef prob_to_rles(x, cutoff=0.5):\n    # From Kaggle\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        #yield rle_encoding(lab_img == i)\n        yield img_to_rle(lab_img == i)\n        \ndef pre_process_image(img):\n    # generation of all 3 image inputs\n    X3 = img\n    X1 = prepare_image_final(X3)\n    X2 = cv2.adaptiveThreshold(X1.astype(np.uint8), 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2)\n        \n    # resize to 256 x 256 for prediction\n    X1 = cv2.resize(X1, (256, 256))\n    X2 = cv2.resize(X2, (256, 256))\n    X3 = cv2.resize(X3, (256, 256))\n    \n    # transforamtion to proper type/shape\n    X1 = np.expand_dims(X1, 3)\n    X2 = np.expand_dims(X2, 3)\n    X1 = np.expand_dims(X1, 0).astype(np.float32)\n    X2 = np.expand_dims(X2, 0).astype(np.float32)\n    X3 = np.expand_dims(X3, 0).astype(np.float32)\n    \n    # normalize\n    X1 /= 255.\n    X2 /= 255.\n    X3 /= 255.\n    \n    return [X1, X2, X3]\n\ndef show(img, cmap=\"gray\", title=None):\n    plt.figure(figsize=(6,6))\n    plt.imshow(img, cmap=cmap)\n    if title is not None:\n        plt.savefig(\"img/{}.png\".format(title))\n    plt.show()","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"ffd7aff4-967c-42f6-9653-f906e017da1e","_uuid":"3bbec988f0ac983e11f4e364543257ada76edc10"},"source":"# Exploration\n\nFor the exploration, I'll only share informations and not provide all the code as it is pretty easy. Feel free to contact me if you want link to the full notebook.\n\n### Type of images\n\nWe have 3 kind of images, Black with White cells, White with Black cells or colored images\n\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/img_type.png?raw=true\">\n\n### Dimensions of images\n\nMost of images have a small ration W/H and have often the same size. We can see a scatter plot of dimensions. The size of the point reflect the number of images\n\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/dimensions.png?raw=true\">"},{"cell_type":"markdown","metadata":{"_cell_guid":"56411e9a-7cf4-4067-8fc0-272f7648dec8","_uuid":"1258ba02aa7f9e6474e1a13de007379639b8e589"},"source":"# Pre-Processing\n\n### Merge of masks\n\nThis part will be skipped but you can find code on several other Kernels\n\n### Image preparation\n\nIt's time now to prepare the 3 kind of images to feed the U-Net Extended. \n<ul>\n<li>The first one is quite simple, it's the native image</li><li>The second one is a B&W version of images with boosted contrast</li><li>The last one is a version with adaptative threshold on the previous image</li>\n</ul>\nAll resize will be done to 256 x 256 for memory reasons\n\nWe can see below the 3 images we will have for 4 differents example (1 example per column). The first row is the pre-processed image, the second one is the result of adaptative threshold and the last one is origianl image.\n\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/pre-process-extended.png?raw=true\">"},{"cell_type":"code","metadata":{"_cell_guid":"79d8a78a-d486-432d-93f0-d8571823d7c2","_uuid":"9ec68092148210af39db5c4078edd01a5592f3b9","collapsed":true},"source":"start = time.time()\n\nW, H = 256, 256\n\nX1 = np.zeros((670, H, W, 1), dtype=np.uint8)\nX2 = np.zeros((670, H, W, 1), dtype=np.uint8)\nX3 = np.zeros((670, H, W, 3), dtype=np.uint8)\ny = np.zeros((670, H, W, 1), dtype=np.uint8)\n\nfor idx, img_path in enumerate(glob.glob(\"kaggle_folder/images/*.png\")):\n    img_name = os.path.basename(img_path)\n    mask_path = os.path.join(\"kaggle_folder/masks/\", img_name)\n    \n    image = cv2.imread(img_path).astype(np.uint8)\n    image2 = prepare_image_final(image).astype(np.uint8)\n    image3 = cv2.adaptiveThreshold(image2, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2).astype(np.uint8)\n\n    mask = cv2.imread(mask_path)\n    \n    image = cv2.resize(image, (W, H))\n    image2 = cv2.resize(image2, (W, H))\n    image3 = cv2.resize(image3, (W, H))\n    mask = cv2.resize(mask[:, :, 0], (W, H))\n            \n    X1[idx, :, :, 0] = image2\n    X2[idx, :, :, 0] = image3\n    X3[idx] = image\n    y[idx, :, :, 0] = mask","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"c5275a8e-3267-41e4-9a22-1a51489d4187","_uuid":"e9581e20cce97c2cd4422908a7ec750263824ecd"},"source":"# Model\n\nNow we have our matrices, let's build the model. As mentionned on the inttroduction, the difference with the U-net model, is a multi convolution in parrallel. It can be represented as follow :\n\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/extended_unet.png?raw=true\">"},{"cell_type":"code","metadata":{"_cell_guid":"d6bb8077-f248-40b9-9a5c-2c8d3da421a4","_uuid":"b57b5255e9aa382f835d36758181e0e5216937c8","collapsed":true},"source":"K_REG = None\nBIAS_INIT = \"zeros\"\nKERNEL_INIT = \"he_normal\"\nACTIVATION = \"relu\"\nIMG_HEIGHT = 256\nIMG_WIDTH = 256\n# Convolution 1\n\ninputs = Input((IMG_HEIGHT, IMG_WIDTH, 1))\n\nc1 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (inputs)\nc1 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c1)\np1 = MaxPooling2D((2, 2)) (c1)\n\nc2 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p1)\nc2 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c2)\np2 = MaxPooling2D((2, 2)) (c2)\n\nc3 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p2)\nc3 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c3)\np3 = MaxPooling2D((2, 2)) (c3)\n\nc4 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p3)\nc4 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c4)\np4 = MaxPooling2D(pool_size=(2, 2)) (c4)\n\n# Convolution 2\n\ninputs2 = Input((IMG_HEIGHT, IMG_WIDTH, 1))\n\nc12 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (inputs2)\nc12 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c12)\np12 = MaxPooling2D((2, 2)) (c12)\n\nc22 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p12)\nc22 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c22)\np22 = MaxPooling2D((2, 2)) (c22)\n\nc32 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p22)\nc32 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c32)\np32 = MaxPooling2D((2, 2)) (c32)\n\nc42 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p32)\nc42 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c42)\np42 = MaxPooling2D(pool_size=(2, 2)) (c42)\n\n# Convolution 3\n\ninputs3 = Input((IMG_HEIGHT, IMG_WIDTH, 3))\n\nc13 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (inputs3)\nc13 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c13)\np13 = MaxPooling2D((2, 2)) (c13)\n\nc23 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p13)\nc23 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c23)\np23 = MaxPooling2D((2, 2)) (c23)\n\nc33 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p23)\nc33 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c33)\np33 = MaxPooling2D((2, 2)) (c33)\n\nc43 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p33)\nc43 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c43)\np43 = MaxPooling2D(pool_size=(2, 2)) (c43)\n\n### Deconvolution\n\np5 = concatenate([p4, p42, p43])\n\nc5 = Conv2D(128, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (p5)\nc5 = Conv2D(128, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c5)\n\nu6 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c5)\nu6 = concatenate([u6, c4, c42, c43])\nc6 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (u6)\nc6 = Conv2D(64, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c6)\n\nu7 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c6)\nu7 = concatenate([u7, c3, c32, c33])\nc7 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (u7)\nc7 = Conv2D(32, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c7)\n\nu8 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c7)\nu8 = concatenate([u8, c2, c22, c23])\nc8 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (u8)\nc8 = Conv2D(16, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c8)\n\nu9 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same') (c8)\nu9 = concatenate([u9, c1, c12, c13], axis=3)\nc9 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (u9)\nc9 = Conv2D(8, (3, 3), activation=ACTIVATION, padding='same', kernel_initializer=KERNEL_INIT, bias_initializer = BIAS_INIT, kernel_regularizer=K_REG) (c9)\n\noutputs = Conv2D(1, (1, 1), activation='sigmoid') (c9)\n\nmodel = Model(inputs=[inputs, inputs2, inputs3], outputs=[outputs])\n\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics = ['binary_crossentropy'])","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"e2b3cb75-bfc1-4420-b9d6-266fb41f8e0e","_uuid":"0fe1672ca2f609ba3a49fad55ecf6aa864c6eeba"},"source":"I prepared also a Callback which create a prediction at every Epochs and generate a git at the end to \"see\" the stability. To simplify the kernel, I won't apply it here but feel free to ask also and I'tt provide you the code :). Now, let's setup ou ImageGenerator"},{"cell_type":"code","metadata":{"_cell_guid":"a87c98f1-a8ee-4f74-955d-e533e7244ca1","_uuid":"c83ea6b7eb14b5f40f65e972e90d84756149a3f0","collapsed":true},"source":"def batch_train(train_generator):\n    img_gen, mask_gen = train_generator\n    while True:\n        x1 = next(img_gen[0])\n        x2 = next(img_gen[1])\n        x3 = next(img_gen[2])\n        m =  next(mask_gen[0])\n        #if x1.shape[0] == x2.shape[0] and x1.shape[0] == x3.shape[0] and x1.shape[0] == m.shape[0]:\n        yield [x1, x2, x3], [m]\n\ndef batch_test(test_generator):\n    img_gen, mask_gen = test_generator\n    while True:\n        x1 = next(img_gen[0])\n        x2 = next(img_gen[1])\n        x3 = next(img_gen[2])\n        m =  next(mask_gen[0])\n        #if x1.shape[0] == x2.shape[0] and x1.shape[0] == x3.shape[0] and x1.shape[0] == m.shape[0]:\n        yield [x1, x2, x3], [m]\n\n\ndef create_generator(X1_train, X1_test, X2_train, X2_test, X3_train, X3_test, y_train, y_test, BATCH_SIZE, seed = 42):\n    # data augmentation doesn't perform as expected, so it's removed\n    data_gen_args_train = {\n        # \"rotation_range\" : 90.,\n        # \"width_shift_range\" : 0.1,\n        # \"height_shift_range\" : 0.1,\n        # \"zoom_range\" : 0.2,\n        # \"shear_range\" : 0.2,\n        # \"fill_mode\" : \"nearest\",\n        # \"cval\" : 0,\n        \"horizontal_flip\" : True,\n        \"vertical_flip\" : True,\n        \"data_format\" : \"channels_last\",\n        \"rescale\" : 1./255                      # convert uint8 to float32\n    }\n\n    data_gen_args_test = {\n        \"horizontal_flip\" : False,\n        \"vertical_flip\" : False,\n        \"data_format\" : \"channels_last\",\n        \"rescale\" : 1./255                      # convert uint8 to float32\n    }\n\n    # Creation of image data augmenter for train set\n    image1_datagen_train = ImageDataGenerator(**data_gen_args_train)\n    image2_datagen_train = ImageDataGenerator(**data_gen_args_train)\n    image3_datagen_train = ImageDataGenerator(**data_gen_args_train)\n    mask_datagen_train = ImageDataGenerator(**data_gen_args_train)\n\n    # fit this generator to train images\n    image1_datagen_train.fit(X1_train, augment=True, seed=seed)\n    image2_datagen_train.fit(X2_train, augment=True, seed=seed)\n    image3_datagen_train.fit(X3_train, augment=True, seed=seed)\n    mask_datagen_train.fit(y_train, augment=True, seed=seed)\n\n    # create a generator of train images augmented\n    image1_generator_train = image1_datagen_train.flow(X1_train, batch_size=BATCH_SIZE, seed=seed)\n    image2_generator_train = image2_datagen_train.flow(X2_train, batch_size=BATCH_SIZE, seed=seed)\n    image3_generator_train = image3_datagen_train.flow(X3_train, batch_size=BATCH_SIZE, seed=seed)\n    mask_generator_train = mask_datagen_train.flow(y_train, batch_size=BATCH_SIZE, seed=seed)\n\n    # Creation of data augmenter for test set\n    image1_datagen_test = ImageDataGenerator(**data_gen_args_test)\n    image2_datagen_test = ImageDataGenerator(**data_gen_args_test)\n    image3_datagen_test = ImageDataGenerator(**data_gen_args_test)\n    mask_datagen_test = ImageDataGenerator(**data_gen_args_test)\n\n    # fit this generator to test images\n    image1_datagen_test.fit(X1_test, augment=True, seed=seed)\n    image2_datagen_test.fit(X2_test, augment=True, seed=seed)\n    image3_datagen_test.fit(X3_test, augment=True, seed=seed)\n    mask_datagen_test.fit(y_test, augment=True, seed=seed)\n\n    # create a generator of test images augmented\n    image1_generator_test = image1_datagen_test.flow(X1_test, batch_size=BATCH_SIZE, seed=seed)\n    image2_generator_test = image2_datagen_test.flow(X2_test, batch_size=BATCH_SIZE, seed=seed)\n    image3_generator_test = image3_datagen_test.flow(X3_test, batch_size=BATCH_SIZE, seed=seed)\n    mask_generator_test = mask_datagen_test.flow(y_test, batch_size=BATCH_SIZE, seed=seed)\n\n    train_generator = [[image1_generator_train, image2_generator_train, image3_generator_train], [mask_generator_train]]\n    test_generator = [[image1_generator_test, image2_generator_test, image3_generator_test], [mask_generator_test]]\n\n    gen_train = batch_train(train_generator)\n    gen_test = batch_test(test_generator)\n\n    return gen_train, gen_test","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"0bebc422-6d57-483a-a86a-ec6941428fa5","_uuid":"2db50ddae339a2c3d84b7a17596a08707c1aad8c"},"source":"Now we can prepare our train and test dataset"},{"cell_type":"code","metadata":{"_cell_guid":"e7f9aec9-fb98-41c9-aa11-f93a23e68b9b","_uuid":"93f50179473b74aadb638c18e11c5277bb382cbb","collapsed":true},"source":"X1_train, X1_test, X2_train, X2_test, X3_train, X3_test, y_train, y_test = train_test_split(X1, X2, X3, y, test_size=0.1, random_state=42)\n\nprint(\"Pre-processed images\")\nprint(X1_train.shape, \" - \",  X1_train.dtype)\nprint(X1_test.shape, \" - \",  X1_test.dtype)\n\nprint(\"\\nAdaptated Threshold images\")\nprint(X2_train.shape, \" - \",  X2_train.dtype)\nprint(X2_test.shape, \" - \",  X2_test.dtype)\n\nprint(\"\\nInitial images\")\nprint(X3_train.shape, \" - \", X3_train.dtype)\nprint(X3_test.shape, \" - \", X3_test.dtype)\n\nprint(\"\\nmask\")\nprint(y_train.shape, \" - \", y_train.dtype)\nprint(y_test.shape, \" - \", y_test.dtype)","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"04a25c7a-dbd3-423d-98be-91d1e75aef60","_uuid":"c3667cffb8358fe78115deb08305e230d55bc3d6"},"source":"Let's now setup our training "},{"cell_type":"code","metadata":{"_cell_guid":"8834caed-d281-454c-a0b0-b1d2ee3accd4","_uuid":"324afd7958990bfbf1f4554e920abe28d4b9bcc8","collapsed":true},"source":"h, w, c = X1_train.shape[1:]\nBATCH_SIZE = 15\nNB_IMAGES_TRAIN = X1_train.shape[0]\nNB_IMAGES_TEST = X1_test.shape[0]\nEPOCHS = 10\nSEED = 42\n\ntrain_generator, test_generator = create_generator(X1_train, X1_test, X2_train, X2_test, X3_train, X3_test, y_train, y_test, BATCH_SIZE=BATCH_SIZE, seed=SEED)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"e3f586ad-b03f-4cf4-b869-5bab7437c05c","_uuid":"c0fbcd7c305b2eb6d1d9cf4e91efabb2c64cb655","collapsed":true},"source":"model.fit_generator(generator = train_generator,\n                    steps_per_epoch = NB_IMAGES_TRAIN//BATCH_SIZE,\n                    epochs=EPOCHS,\n                    validation_data=test_generator,\n                    validation_steps = NB_IMAGES_TEST//BATCH_SIZE\n                    )\n\nmodel.save(\"model_kaggle.h5\")","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"dcbcc75a-b377-464c-a627-d37e9aed97a9","_uuid":"5ebb043c9291fe96d9eb984e6f6758d658359ea1"},"source":"# Fine-Tune Threshold\n\nEvery pixels are out from a sigmoid. Based on the threshold, we will filter more or less pixels. In order to maximise our score, we will check with several threshold the average number of mask above a certain value. Compare to Kaggle's evaluation, this will be done on the full mask. The post-processing will split the prediction to multiple masks"},{"cell_type":"code","metadata":{"_cell_guid":"33797375-d81c-4326-be9c-03d60b6a56e3","_uuid":"a04b1256c8386d341d3604d32cb8be9974551b7f","collapsed":true},"source":"result_trigger = []\nfor trigger in [0.05 * i for i in range(5, 15)]:  # this is to test ou threshold for a pixel to be B or W\n    print(\"Trigger = {:.3f}\".format(trigger))\n    final_score = []\n    threshold = [0.05 * i for i in range(10, 20)] # for each full mask having a IoU > threshold\n    for t in threshold:\n        score = []\n        for idx in range(NB_IMAGES_TEST):\n            X1 = X1_test[idx].astype(np.float64)\n            X2 = X2_test[idx].astype(np.float64)\n            X3 = X3_test[idx].astype(np.float64)\n\n            X1 /= 255.\n            X2 /= 255.\n            X3 /= 255.\n\n            X1 = np.expand_dims(X1, 0)\n            X2 = np.expand_dims(X2, 0)\n            X3 = np.expand_dims(X3, 0)\n\n            y_pred = model.predict([X1, X2, X3])\n\n            mask = y_test[idx, :, : , 0]\n            pred = y_pred[0, :, :, 0]\n            \n            # I didn't consider the rescale yet\n\n            IOU = compute_IoU(pred, mask, trigger=trigger)  # if predicition of a specific pixel is above trigger, it is white\n            if IOU >= t:\n                score.append(1)\n            else:\n                score.append(0)\n        final_score.append(sum(score)/len(score))\n    print(\"Final score = {:.3f}\".format(sum(final_score)/len(final_score)))\n    result_trigger.append(sum(final_score)/len(final_score))","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"b93a455e-705c-4303-a070-092dc81bd960","_uuid":"fd32bc5059d9107b315308f0c7aec4814cf2c7fa"},"source":"The peak is at 0.45 but it's beter to predict a bit more pixel to have maybe a bit more borders (it's better to predict a bit too much than not enought is IoU). So we will keep 0.4"},{"cell_type":"markdown","metadata":{"_cell_guid":"69a72969-2812-4ac3-8149-0adee86d4450","_uuid":"d01cddab16f15ecc5b8527ccaf081935d55863f8"},"source":"# Post Processing\n\nIdeas from this phase comes from <a href=\"https://www.kaggle.com/ahassaine/pure-image-processing-lb-0-274\">this Kernel</a> and <a href=\"https://docs.opencv.org/3.3.1/d3/db4/tutorial_py_watershed.html\">this tutorial</a>. Let's first check the prediction and explore the post-processing on a fixed prediction (from test set)"},{"cell_type":"code","metadata":{"_cell_guid":"8948ea43-2adb-49cd-8620-80a7b640157f","_uuid":"8cce7c13edcc94efb7d6367734d110b92fbb619e","collapsed":true},"source":"path = \"images/test/0a849e0eb15faa8a6d7329c3dd66aabe9a294cccb52ed30a90c8ca99092ae732/images/0a849e0eb15faa8a6d7329c3dd66aabe9a294cccb52ed30a90c8ca99092ae732.png\"\nimg_name = \"0a849e0eb15faa8a6d7329c3dd66aabe9a294cccb52ed30a90c8ca99092ae732\"\n\n# path = \"images/test/1879f4f4f05e2bada0ffeb46c128b8df7a79b14c84f38c3e216a69653495153b/images/1879f4f4f05e2bada0ffeb46c128b8df7a79b14c84f38c3e216a69653495153b.png\"\n# img_name = \"1879f4f4f05e2bada0ffeb46c128b8df7a79b14c84f38c3e216a69653495153b\"\n\nimg = cv2.imread(path)\nW, H = img.shape[:-1]\nX = pre_process_image(img)\ny = model.predict(X)\ny = (y>0.4).astype(np.uint8)   # the best threshold is 0.4\ny = cv2.resize(y[0, :, :, 0], (H, W))\n\nshow(y)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"f7efd169-a3fb-43de-9be9-17f1fd965a25","_uuid":"d89457b8a6bce982271e38c84d36ff0cc9a5b2d5","collapsed":true},"source":"with_morph = False\nimg = cv2.imread(path)\n\n# show(y)\n\n# noise removal\nkernel = np.ones((3,3), np.uint8)\nopening = cv2.morphologyEx(y, cv2.MORPH_OPEN, kernel, iterations=2)\nopening = opening*255\n# show(opening)\n\n# opening = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, np.ones((25,25), np.uint8))\n\nstatus=\"off\"\nif with_morph:\n    circle7=cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(7, 7))\n    opening=cv2.morphologyEx(opening, cv2.MORPH_OPEN, circle7)\n    status=\"on\"\n\n# sure background area\nsure_bg = cv2.dilate(opening, kernel, iterations=1)\n# show(sure_bg)\n\n# Finding sure foreground area\ndist_transform = cv2.distanceTransform(opening, cv2.DIST_C, cv2.DIST_MASK_PRECISE)\n# show(dist_transform)\n\nret, sure_fg = cv2.threshold(dist_transform, 0.1*dist_transform.max(), 255, 0)\n# show(sure_fg)\n\n# Finding unknown region\nsure_fg = np.uint8(sure_fg)\nunknown = cv2.subtract(sure_bg, sure_fg)\n# show(unknown)\n\n# Marker labelling\nret, markers = cv2.connectedComponents(sure_fg)\n\n# Add one to all labels so that sure background is not 0, but 1\nmarkers = markers+1\n\n# Now, mark the region of unknown with zero\nmarkers[unknown==255] = 0\n# show(markers, \"jet\")\n\n# \nmarkers = cv2.watershed(img, markers)\nimg[markers == -1] = [255,0,0]\nshow(markers, \"jet\")\nshow(img, \"gray\")","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"54003210-3634-4441-9c21-819f36b2b0f6","_uuid":"9ebdd18f36a027d4ea54ed965685bf5f65832df5"},"source":"We can see that the prediction is not so bad. Let's now prepare our prediction on Validation images"},{"cell_type":"markdown","metadata":{"_cell_guid":"ec30538a-5989-42d1-a0fa-14eff80856c0","_uuid":"d8426d78eb1f1940f478139d4662bc63ff849090"},"source":"# Prediction\n\nNow we have our model, our pre/post processing so we can run the prediction"},{"cell_type":"code","metadata":{"_cell_guid":"ea34addf-95a2-44a1-802c-d13bb7a0e2e2","_uuid":"c55348b36c3581db25436ca7d4f60e3db285dde2","collapsed":true},"source":"img_list = []\nrle_list = []\nkernel = np.ones((3,3), np.uint8)\n    \nfor img in glob.glob(\"images/test/*/images/*.png\"):\n    img_name = os.path.basename(img)[:-4]\n    img = cv2.imread(img)\n    W, H, L = img.shape\n    \n    # now we have our image, let's predict the mask\n    X = pre_process_image(img)\n    y = model.predict(X)\n    y = (y>0.4).astype(np.uint8)\n    \n    # let's resize the image to original size\n    y = cv2.resize(y[0, :, :, 0], (H, W))\n    \n    # now we can do the post-processing shown above\n    opening = cv2.morphologyEx(y, cv2.MORPH_OPEN, kernel, iterations=2)\n    opening = opening*255\n\n    circle7=cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(7, 7))\n    opening=cv2.morphologyEx(opening, cv2.MORPH_OPEN, circle7)\n    \n    sure_bg = cv2.dilate(opening, kernel, iterations=1)\n    dist_transform = cv2.distanceTransform(opening, cv2.DIST_C, cv2.DIST_MASK_PRECISE)\n    ret, sure_fg = cv2.threshold(dist_transform, 0.2*dist_transform.max(), 255, 0)\n    sure_fg = np.uint8(sure_fg)\n    unknown = cv2.subtract(sure_bg, sure_fg)\n    ret, markers = cv2.connectedComponents(sure_fg)\n    markers = markers+1\n    markers[unknown==255] = 0\n    markers = cv2.watershed(img, markers)\n    \n    # now for each cells, let's create the RLE encoding\n    max_cells = markers.max()\n    for i in range(2, max_cells+1):\n        mask = (markers==i).astype(np.uint8)\n        rle = img_to_rle(mask)\n        if len(rle)>0:\n            img_list.append(img_name)\n            rle_list.append(rle)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"c1167e98-5adc-4af8-96bb-6c9bf24d01cd","_uuid":"3a67368625634dfdcc3b0363650fbca3ea99c880","collapsed":true},"source":"# and finally save the dataframe !\ndf = pd.DataFrame()\ndf['ImageId'] = img_list\ndf['EncodedPixels'] = pd.Series(rle_list).apply(lambda x: ' '.join(str(y) for y in x))\ndf.to_csv('sub-kaggle_model_extended.csv', index=False)","execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"80c8aed9-f31d-46aa-a9da-c67cabd692aa","_uuid":"a51d3656dd515eab158f2c7320c7f391059a0b88","collapsed":true},"source":"stop = time.time()\nprint(\"Time to run the kernel : {:02f}s\".format(stop-start))","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"c8c8f2f1-c5c5-4b78-91e7-a3612763a564","_uuid":"b1f1287ae64e6afaa2c0e2696cb3d4d4cd3e634f"},"source":"# Bonus\n\nDuring training of Classic Unet and Extended one, I've saved predictions and generated a gif of the training for 4 differents test images. Let's compare both  "},{"cell_type":"markdown","metadata":{"_cell_guid":"31555e28-efb9-462d-98d8-f0147d2d5e7e","_uuid":"58f5691d1f5a8e3001477461cfe992902f15e270"},"source":"<p style=\"float: left; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet-8.gif?raw=true\" style=\"width: 100%\">\n</p>\n\n<p style=\"float: right; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet_extended-8.gif?raw=true\" style=\"width: 100%\">\n</p>"},{"cell_type":"markdown","metadata":{"_cell_guid":"bd25cda0-ad14-44b3-be49-bac4b916058f","_uuid":"3b443a723054b5378af3d3e758097a673d1bfe7c"},"source":"<p style=\"float: left; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet-11.gif?raw=true\" style=\"width: 100%\">\n</p>\n\n<p style=\"float: right; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet_extended-11.gif?raw=true\" style=\"width: 100%\">\n</p>"},{"cell_type":"markdown","metadata":{"_cell_guid":"a1d5bfec-93f6-4c93-a3fa-d494a9c5c9ac","_uuid":"dde1676a634cbd53cb01ee9ef2b25e54a05fc43a"},"source":"<p style=\"float: left; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet-12.gif?raw=true\" style=\"width: 100%\">\n</p>\n\n<p style=\"float: right; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet_extended-12.gif?raw=true\" style=\"width: 100%\">\n</p>"},{"cell_type":"markdown","metadata":{"_cell_guid":"77cee3eb-65f0-4f42-9b05-bfe65d1e9df0","_uuid":"e9bb9014d77f776a2b170c8dff19f08f6dfb7583"},"source":"<p style=\"float: left; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet-24.gif?raw=true\" style=\"width: 100%\">\n</p>\n\n<p style=\"float: right; font-size: 9pt; text-align: center; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\">\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/training/training_unet_extended-24.gif?raw=true\" style=\"width: 100%\">\n</p>"},{"cell_type":"markdown","metadata":{"_cell_guid":"fe8c67e1-0e3b-4674-b6a9-8e9164de5fde","_uuid":"ce53df067c0190cf75e5d48eeca8267200d15b96"},"source":"We can see that the training stabilize clearly faster. Unfortunately some prediction are really bad and decrease a lot the score as only few images are taken in the evaluation (cf the image below :)\n\n<img src=\"https://github.com/Coni63/OC_DS/blob/master/P9/img/extraction_mdl_1/0f1f896d9ae5a04752d3239c690402c022db4d72c0d2c087d73380896f72c466.png?raw=true\"/>"}],"nbformat":4,"nbformat_minor":1}