{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This is just draft kernel, where I have tested different staff I was interested in trying.\n# That's why it looks really trashy, so if you don't want to harm your eyes, then just skip it :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport random\nimport warnings\nimport zipfile\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom itertools import chain\nfrom skimage.io import imread, imshow, imread_collection, concatenate_images\nfrom skimage.transform import resize\nfrom skimage.morphology import label\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## As we have zip files first I need to unzip my folders with images\n### Here I specify my pathes to necessary folders"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Unzipping some files\nstage1_train = 'stage1_train'\nstage1_test = 'stage1_test'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now I will unzip them to the output folder"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Will unzip the files so that you can see them..\nwith zipfile.ZipFile(\"../input/data-science-bowl-2018/\"+stage1_train+\".zip\",\"r\") as z:\n    z.extractall(stage1_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Will unzip the files so that you can see them..\nwith zipfile.ZipFile(\"../input/data-science-bowl-2018/\"+stage1_test+\".zip\",\"r\") as z:\n    z.extractall(stage1_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Now let's specify some constants that we will use "},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATH = '../working/stage1_train/'\nTEST_PATH = '../working/stage1_test/'\nIMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3\n\nwarnings.filterwarnings('ignore', category=UserWarning, module='skimage')\nseed = 42\nrandom.seed = seed\nnp.random.seed = seed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Getting images' ids"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get train and test IDs\ntrain_ids = next(os.walk(TRAIN_PATH))[1]\ntest_ids = next(os.walk(TEST_PATH))[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\nimport sys\nimport warnings\nimport numpy as np\nfrom itertools import chain\nfrom skimage.io import imread, imshow, imread_collection, concatenate_images\nfrom skimage.transform import resize\nfrom skimage.morphology import label\nfrom keras.utils import Progbar","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_data(IMG_WIDTH=256, IMG_HEIGHT=256, IMG_CHANNELS=3):\n    X_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\n    Y_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n    print('Getting and resizing train images and masks ... ')\n    sys.stdout.flush()\n    if os.path.isfile('train_img.npy') and os.path.isfile('train_mask.npy'):\n        print('Train file loaded from memory')\n        X_train = np.load('train_img.npy')\n        Y_train = np.load('train_mask.npy')\n        return X_train, Y_train\n    a = Progbar(len(train_ids))\n    for n, id_ in enumerate(train_ids):\n        path = TRAIN_PATH + id_\n        img = imread(path + '/images/' + id_ + '.png')[:, :, :IMG_CHANNELS]\n        img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n        X_train[n] = img\n        mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n        for mask_file in next(os.walk(path + '/masks/'))[2]:\n            mask_ = imread(path + '/masks/' + mask_file)\n            mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant',\n                                        preserve_range=True), axis=-1)\n            mask = np.maximum(mask, mask_)\n        Y_train[n] = mask\n        a.update(n)\n    np.save('train_img', X_train)\n    np.save('train_mask', Y_train)\n    return X_train, Y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_data(IMG_WIDTH=256, IMG_HEIGHT=256, IMG_CHANNELS=3):\n    X_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\n    sizes_test = []\n    print('\\nGetting and resizing test images ... ')\n    sys.stdout.flush()\n    if os.path.isfile('test_img.npy') and os.path.isfile('test_size.npy'):\n        print('Test file loaded from memory')\n        X_test = np.load('test_img.npy')\n        sizes_test = np.load('test_size.npy')\n        return X_test,sizes_test\n    b = Progbar(len(test_ids))\n    for n, id_ in enumerate(test_ids):\n        path = TEST_PATH + id_\n        img = imread(path + '/images/' + id_ + '.png')[:, :, :IMG_CHANNELS]\n        sizes_test.append([img.shape[0], img.shape[1]])\n        img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n        X_test[n] = img\n        b.update(n)\n    np.save('test_img', X_test)\n    np.save('test_size', sizes_test)\n    return X_test,sizes_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get train_data\ntrain_img, train_mask = get_train_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get test_data\ntest_img, test_img_sizes = get_test_data()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining U-Net model and metrics\n## As a metric I will use Dice Coefficient"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose,Convolution2D\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras import backend as K\n\nsmooth = 1.\n\n\n# Metric function\ndef dice_coef(y_true, y_pred):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\n\ndef make_model(IMG_WIDTH=256,IMG_HEIGHT=256,IMG_CHANNELS=3):\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = Lambda(lambda x: x / 255)(inputs)\n    c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(s)\n    c1 = Dropout(0.1)(c1)\n    c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c1)\n    p1 = MaxPooling2D((2, 2))(c1)\n    c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p1)\n    c2 = Dropout(0.1)(c2)\n    c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c2)\n    p2 = MaxPooling2D((2, 2))(c2)\n\n    c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p2)\n    c3 = Dropout(0.2)(c3)\n    c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c3)\n    p3 = MaxPooling2D((2, 2))(c3)\n\n    c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p3)\n    c4 = Dropout(0.2)(c4)\n    c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c4)\n    p4 = MaxPooling2D(pool_size=(2, 2))(c4)\n\n    c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p4)\n    c5 = Dropout(0.3)(c5)\n    c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c5)\n\n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u6)\n    c6 = Dropout(0.2)(c6)\n    c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c6)\n\n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u7)\n    c7 = Dropout(0.2)(c7)\n    c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c7)\n\n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u8)\n    c8 = Dropout(0.1)(c8)\n    c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c8)\n\n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u9)\n    c9 = Dropout(0.1)(c9)\n    c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c9)\n\n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nu_net = make_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preliminary training on 50 epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nu_net.fit(train_img, train_mask, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predicting masks for test images"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Predicting\")\n# Predict on test data\ntest_mask = u_net.predict(test_img, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Obtained mask"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(test_mask[0].reshape(256,256))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_img.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Actual image"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(test_img[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now, as far as I now I can not estimate quality of my masks for test images using Dice Coefficient, because I don't have ground truth masks.\n### So what I am going to do, I will split my train set into train/validation set, and use this validation set as test set, predict masks for it, and then evaluate it."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(train_img, train_mask, test_size=0.15, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Defining another model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nu_net_1 = make_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nu_net_1.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Predicting\")\n# Predict on test data\ntest_mask_validation = u_net_1.predict(X_valid, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predicted mask"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(test_mask_validation[0].reshape(256,256))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ground truth mask"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(y_valid[0].reshape(256,256))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now let's evaluate our model"},{"metadata":{"trusted":true},"cell_type":"code","source":"u_net_1.evaluate(X_valid,y_valid, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"u_net_1.metrics_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dice_score = 0\nfor i in range(len(test_mask_validation)):\n    dice_score += dice_coef(test_mask_validation[i].reshape(256,256),y_valid[i].reshape(256,256).astype(\"float32\")).numpy()\nprint(dice_score/len(test_mask_validation))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Now let's try adding some early stopping"},{"metadata":{},"cell_type":"markdown","source":"## Monitoring dice_coef"},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    EarlyStopping(monitor='dice_coef', patience=10, verbose=1),\n    ReduceLROnPlateau(monitor='dice_coef', factor=0.1, patience=3, min_lr=0.000001, verbose=1),\n    ModelCheckpoint('model_weights_1.h5', verbose=1, save_best_only=True, save_weights_only=True)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nu_net_es = make_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = u_net_es.fit(X_train, y_train, batch_size=16, epochs=100, callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Monitoring loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks1 = [\n    EarlyStopping(monitor='loss', patience=10, verbose=1),\n    ReduceLROnPlateau(monitor='loss', factor=0.1, patience=3, min_lr=0.000001, verbose=1),\n    ModelCheckpoint('model_weights_1.h5', verbose=1, save_best_only=True, save_weights_only=True)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nu_net_es_loss = make_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = u_net_es_loss.fit(X_train, y_train, batch_size=16, epochs=200, callbacks=callbacks1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nplt.title(\"Learning curve\")\nplt.plot(results.history[\"loss\"], label=\"loss\")\nplt.plot(results.history[\"dice_coef\"], label=\"dice_coef\")\nplt.plot( np.argmin(results.history[\"dice_coef\"]), np.min(results.history[\"dice_coef\"]), marker=\"x\", color=\"r\", label=\"best model\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"log_loss\")\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose,Convolution2D\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\n\n\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\n\ndef conv2d_block(input_tensor, n_filters, kernel_size=3, dropout=1, activation='relu'):\n    # first layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(input_tensor)\n    c1 = Dropout(0.1)(c1)\n    # second layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(c1)\n    return c1\n\n\ndef unet_model(IMG_WIDTH=256,IMG_HEIGHT=256,IMG_CHANNELS=3, activation='relu', dropout=0.1):\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = Lambda(lambda x: x / 255)(inputs)\n    n_filters = 16\n    \n    c1 = conv2d_block(s, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    p1 = MaxPooling2D((2, 2), name='pool1')(c1)\n    \n    \n    c2 = conv2d_block(p1, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n    p2 = MaxPooling2D((2, 2))(c2)\n\n    c3 = conv2d_block(p2, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n    p3 = MaxPooling2D((2, 2))(c3)\n\n    c4 = conv2d_block(p3, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n    p4 = MaxPooling2D((2, 2))(c4)\n\n    c5 = conv2d_block(p4, 256, kernel_size=3, dropout=dropout*3, activation=activation)\n\n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = conv2d_block(u6, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = conv2d_block(u7, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = conv2d_block(u8, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = conv2d_block(u9, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(train_img, train_mask, test_size=0.15, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Testing our model with different activation functions "},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_relu = unet_model(activation=\"relu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nunet_relu.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_relu.evaluate(X_valid,y_valid, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Elu activation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu = unet_model(activation=\"elu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nunet_elu.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu.evaluate(X_valid,y_valid, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Trying adding validation during learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_valid = unet_model(activation=\"elu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_valid.fit(X_train, y_train, batch_size=16, epochs=50, validation_data=(X_valid, y_valid))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Adding callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks_new = [\n    EarlyStopping(monitor='dice_coef', mode='max', patience=10, verbose=1),\n    ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1),\n    ModelCheckpoint('model_weights.h5', verbose=1, save_best_only=True, save_weights_only=True)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_call = unet_model(activation=\"elu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_call.fit(X_train, y_train, batch_size=16, epochs=50, \n                  validation_data=(X_valid, y_valid), callbacks=callbacks_new)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Sigmoid activation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_sigmoid = unet_model(activation=\"sigmoid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nunet_sigmoid.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_sigmoid.evaluate(X_valid,y_valid, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Best accuracy is obtainde using elu activation function"},{"metadata":{},"cell_type":"markdown","source":"## Now let's test different droput values"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_02_dropout = unet_model(activation=\"elu\", dropout=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nunet_elu_02_dropout.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_015_dropout = unet_model(activation=\"elu\", dropout=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nunet_elu_015_dropout.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_015_dropout.evaluate(X_valid,y_valid, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## trying another activation functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_selu_015_dropout = unet_model(activation=\"selu\", dropout=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nTraining...\")\nunet_selu_015_dropout.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Adding learning rate change over time"},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks2 = [\n    ReduceLROnPlateau(monitor='loss', factor=0.1, patience=3, min_lr=0.000001, verbose=1),\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_015_dropout_cbs = unet_model(activation=\"elu\", dropout=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_015_dropout_cbs.fit(X_train, y_train, batch_size=16, epochs=50, callbacks=callbacks2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv2d_block_ccd(input_tensor, n_filters, kernel_size=3, dropout=1, activation='relu'):\n    # first layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(input_tensor)\n    # second layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(c1)\n    c1 = Dropout(0.1)(c1)\n    return c1\n\n\ndef unet_model_ccd(IMG_WIDTH=256,IMG_HEIGHT=256,IMG_CHANNELS=3, activation='relu', dropout=0.1):\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = Lambda(lambda x: x / 255)(inputs)\n    n_filters = 16\n    \n    c1 = conv2d_block_ccd(s, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    p1 = MaxPooling2D((2, 2), name='pool1')(c1)\n    \n    \n    c2 = conv2d_block_ccd(p1, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n    p2 = MaxPooling2D((2, 2))(c2)\n\n    c3 = conv2d_block_ccd(p2, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n    p3 = MaxPooling2D((2, 2))(c3)\n\n    c4 = conv2d_block_ccd(p3, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n    p4 = MaxPooling2D((2, 2))(c4)\n\n    c5 = conv2d_block_ccd(p4, 256, kernel_size=3, dropout=dropout*3, activation=activation)\n\n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = conv2d_block_ccd(u6, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = conv2d_block_ccd(u7, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = conv2d_block_ccd(u8, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = conv2d_block_ccd(u9, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    \n    d10 = Dropout\n\n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_015_dropout_ccd = unet_model_ccd(activation=\"elu\", dropout=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_015_dropout_ccd.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers.core import Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unet_model_ccd_d(IMG_WIDTH=256,IMG_HEIGHT=256,IMG_CHANNELS=3, activation='relu', dropout=0.1):\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = Lambda(lambda x: x / 255)(inputs)\n    n_filters = 16\n    \n    c1 = conv2d_block_ccd(s, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    p1 = MaxPooling2D((2, 2), name='pool1')(c1)\n    \n    \n    c2 = conv2d_block_ccd(p1, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n    p2 = MaxPooling2D((2, 2))(c2)\n\n    c3 = conv2d_block_ccd(p2, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n    p3 = MaxPooling2D((2, 2))(c3)\n\n    c4 = conv2d_block_ccd(p3, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n    p4 = MaxPooling2D((2, 2))(c4)\n\n    c5 = conv2d_block_ccd(p4, 256, kernel_size=3, dropout=dropout*3, activation=activation)\n\n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = conv2d_block_ccd(u6, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = conv2d_block_ccd(u7, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = conv2d_block_ccd(u8, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = conv2d_block_ccd(u9, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    \n    d10 = Dense(256,activation=activation)(c9)\n    d10 = Dense(128,activation=activation)(d10)\n\n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(d10)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get u_net model\nunet_elu_015_dropout_ccd_dd = unet_model_ccd_d(activation=\"elu\", dropout=0.15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_015_dropout_ccd_dd.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_elu_015_dropout_ccd_dd.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's try avgpool instead of max pool"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose,Convolution2D\nfrom keras.layers.pooling import MaxPooling2D, AveragePooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\n\n\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\n\ndef conv2d_block(input_tensor, n_filters, kernel_size=3, dropout=1, activation='relu'):\n    # first layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(input_tensor)\n    c1 = Dropout(0.1)(c1)\n    # second layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(c1)\n    return c1\n\n\ndef unet_model_av_pool(IMG_WIDTH=256,IMG_HEIGHT=256,IMG_CHANNELS=3, activation='relu', dropout=0.1):\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = Lambda(lambda x: x / 255)(inputs)\n    n_filters = 16\n    \n    c1 = conv2d_block(s, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    p1 = AveragePooling2D((2, 2), name='pool1')(c1)\n    \n    \n    c2 = conv2d_block(p1, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n    p2 = AveragePooling2D((2, 2))(c2)\n\n    c3 = conv2d_block(p2, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n    p3 = AveragePooling2D((2, 2))(c3)\n\n    c4 = conv2d_block(p3, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n    p4 = AveragePooling2D((2, 2))(c4)\n\n    c5 = conv2d_block(p4, 256, kernel_size=3, dropout=dropout*3, activation=activation)\n\n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = conv2d_block(u6, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = conv2d_block(u7, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = conv2d_block(u8, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = conv2d_block(u9, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_avpool = unet_model_av_pool(activation='elu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_avpool.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose,Convolution2D\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\n\n\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\n\ndef conv2d_block_nodo(input_tensor, n_filters, kernel_size=3, dropout=1, activation='relu'):\n    # first layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(input_tensor)\n    # second layer\n    c1 = Conv2D(filters=n_filters, activation=activation, kernel_size=(kernel_size, kernel_size), kernel_initializer=\"he_normal\",\n               padding=\"same\")(c1)\n    return c1\n\n\ndef unet_model_nodo(IMG_WIDTH=256,IMG_HEIGHT=256,IMG_CHANNELS=3, activation='relu', dropout=0.1):\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = Lambda(lambda x: x / 255)(inputs)\n    n_filters = 16\n    \n    c1 = conv2d_block_nodo(s, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n    p1 = MaxPooling2D((2, 2), name='pool1')(c1)\n    \n    \n    c2 = conv2d_block_nodo(p1, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n    p2 = MaxPooling2D((2, 2))(c2)\n\n    c3 = conv2d_block_nodo(p2, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n    p3 = MaxPooling2D((2, 2))(c3)\n\n    c4 = conv2d_block_nodo(p3, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n    p4 = MaxPooling2D((2, 2))(c4)\n\n    c5 = conv2d_block_nodo(p4, 256, kernel_size=3, dropout=dropout*3, activation=activation)\n\n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = conv2d_block_nodo(u6, 128, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = conv2d_block_nodo(u7, 64, kernel_size=3, dropout=dropout*2, activation=activation)\n\n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = conv2d_block_nodo(u8, 32, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = conv2d_block_nodo(u9, 16, kernel_size=3, dropout=dropout*1, activation=activation)\n\n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[dice_coef])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_nodo = unet_model_nodo(activation='elu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_nodo.fit(X_train, y_train, batch_size=16, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Trying to load model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import load_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loaded_model = unet_model(activation=\"elu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loaded_model.load_weights(\"model_weights.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_masks = loaded_model.predict(test_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(predicted_masks[0].reshape(256, 256))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(test_img[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}