{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport random\nimport warnings\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom itertools import chain\nfrom skimage.io import imread, imshow, imread_collection, concatenate_images\nfrom skimage.transform import resize\nfrom skimage.morphology import label\nimport skimage.io as io\nimport skimage.transform as trans\n\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.optimizers import *\nfrom keras.callbacks import *\nfrom keras import backend as K\nfrom keras.utils.generic_utils import get_custom_objects\n\nimport tensorflow as tf\n\n# Set some parameters\nIMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3\nTRAIN_PATH = '../input/stage1_train/'\nTEST_PATH = '../input/stage1_test/'\n\nwarnings.filterwarnings('ignore', category=UserWarning, module='skimage')\nseed = 42\nrandom.seed = seed\nnp.random.seed = seed","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Get train and test IDs\ntrain_ids = next(os.walk(TRAIN_PATH))[1]\ntest_ids = next(os.walk(TEST_PATH))[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get and resize train images and masks\nX_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nY_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\nprint('Getting and resizing train images and masks ... ')\nsys.stdout.flush()\nfor n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):\n    path = TRAIN_PATH + id_\n    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_train[n] = img\n    mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n    for mask_file in next(os.walk(path + '/masks/'))[2]:\n        mask_ = imread(path + '/masks/' + mask_file)\n        mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant', \n                                      preserve_range=True), axis=-1)\n        mask = np.maximum(mask, mask_)\n    Y_train[n] = mask\n\n# Get and resize test images\nX_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nsizes_test = []\nprint('Getting and resizing test images ... ')\nsys.stdout.flush()\nfor n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n    path = TEST_PATH + id_\n    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    sizes_test.append([img.shape[0], img.shape[1]])\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_test[n] = img\n\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check if training data looks all right\nix = random.randint(0, len(train_ids))\nimshow(X_train[ix])\nplt.show()\nimshow(np.squeeze(Y_train[ix]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define IoU metric\ndef mean_iou(y_true, y_pred):\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        y_pred_ = tf.to_int32(y_pred > t)\n        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)\n        K.get_session().run(tf.local_variables_initializer())\n        with tf.control_dependencies([up_opt]):\n            score = tf.identity(score)\n        prec.append(score)\n    return K.mean(K.stack(prec), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build U-Net model\n\ninputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\ns = Lambda(lambda x: x / 255) (inputs)\n\nconv1 = Conv2D(64, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(s)\nconv1 = Conv2D(64, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv1)\npool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n\n\nconv2 = Conv2D(128, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(pool1)\nconv2 = Conv2D(128, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv2)\npool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n\nconv3 = Conv2D(256, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(pool2)\nconv3 = Conv2D(256, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv3)\npool3 = MaxPooling2D(pool_size=(2, 2))(conv3)\n\nconv4 = Conv2D(512, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(pool3)\nconv4 = Conv2D(512, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv4)\ndrop4 = Dropout(0.5)(conv4)\npool4 = MaxPooling2D(pool_size=(2, 2))(drop4)\n\nconv5 = Conv2D(1024, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(pool4)\nconv5 = Conv2D(1024, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv5)\ndrop5 = Dropout(0.5)(conv5)\n\nup6 = Conv2D(512, 2, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(drop5))\nmerge6 = concatenate([drop4,up6], axis = 3)\nconv6 = Conv2D(512, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(merge6)\nconv6 = Conv2D(512, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv6)\n\nup7 = Conv2D(256, 2, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv6))\nmerge7 = concatenate([conv3,up7], axis = 3)\nconv7 = Conv2D(256, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(merge7)\nconv7 = Conv2D(256, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv7)\n\nup8 = Conv2D(128, 2, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv7))\nmerge8 = concatenate([conv2,up8], axis = 3)\nconv8 = Conv2D(128, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(merge8)\nconv8 = Conv2D(128, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv8)\n\nup9 = Conv2D(64, 2, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv8))\nmerge9 = concatenate([conv1,up9], axis = 3)\nconv9 = Conv2D(64, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(merge9)\nconv9 = Conv2D(64, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv9)\nconv9 = Conv2D(2, 3, activation = swish, padding = 'same', kernel_initializer = 'he_normal')(conv9)\nconv10 = Conv2D(1, 1, activation = 'sigmoid')(conv9)\n\nmodel = Model(input = inputs, output = conv10)\n\nmodel.compile(optimizer = Adam(lr = 1e-4), loss = 'binary_crossentropy', metrics = [mean_iou, 'accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"upd: relu to elu."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit model\nearlystopper = EarlyStopping(patience=5, verbose=1)\ncheckpointer = ModelCheckpoint('model-dsbowl2018-1.h5', verbose=1, save_best_only=True)\nclr = CyclicLR(base_lr=1e-4, max_lr=0.006,\n                        step_size=2000., mode='triangular2')\n\nresults = model.fit(X_train, Y_train, validation_split=0.1, batch_size=16, epochs=50, \n                    callbacks=[earlystopper, checkpointer, clr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict on train, val and test\nmodel = load_model('model-dsbowl2018-1.h5', custom_objects={'mean_iou': mean_iou})\npreds_train = model.predict(X_train[:int(X_train.shape[0]*0.9)], verbose=1)\npreds_val = model.predict(X_train[int(X_train.shape[0]*0.9):], verbose=1)\npreds_test = model.predict(X_test, verbose=1)\n\n# Threshold predictions\npreds_train_t = (preds_train > 0.5).astype(np.uint8)\npreds_val_t = (preds_val > 0.5).astype(np.uint8)\npreds_test_t = (preds_test > 0.5).astype(np.uint8)\n\n# Create list of upsampled test masks\npreds_test_upsampled = []\nfor i in range(len(preds_test)):\n    preds_test_upsampled.append(resize(np.squeeze(preds_test[i]), \n                                       (sizes_test[i][0], sizes_test[i][1]), \n                                       mode='constant', preserve_range=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Perform a sanity check on some random training samples\nix = random.randint(0, len(preds_train_t))\nimshow(X_train[ix])\nplt.show()\nimshow(np.squeeze(Y_train[ix]))\nplt.show()\nimshow(np.squeeze(preds_train_t[ix]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Perform a sanity check on some random validation samples\nix = random.randint(0, len(preds_val_t))\nimshow(X_train[int(X_train.shape[0]*0.9):][ix])\nplt.show()\nimshow(np.squeeze(Y_train[int(Y_train.shape[0]*0.9):][ix]))\nplt.show()\nimshow(np.squeeze(preds_val_t[ix]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run-length encoding stolen from https://www.kaggle.com/rakhlin/fast-run-length-encoding-python\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cutoff=0.5):\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        yield rle_encoding(lab_img == i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_test_ids = []\nrles = []\nfor n, id_ in enumerate(test_ids):\n    rle = list(prob_to_rles(preds_test_upsampled[n]))\n    rles.extend(rle)\n    new_test_ids.extend([id_] * len(rle))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create submission DataFrame\nsub = pd.DataFrame()\nsub['ImageId'] = new_test_ids\nsub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\nsub.to_csv('sub-dsbowl2018-1.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}