{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# work based on https://github.com/bnsreenu/python_for_microscopists\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport shutil\nfrom tqdm import tqdm\nfrom skimage.io import imread, imshow\nfrom skimage.transform import resize\n\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nTRAINING_PATH = '/kaggle/working/stage1_train/'\nTEST_PATH = '/kaggle/working/stage1_test/'\n\nIMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Unzipping all the train and test dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf /kaggle/working/*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!mkdir /kaggle/working/stage1_train/\n!unzip ../input/data-science-bowl-2018/stage1_train.zip\nfor filename in os.listdir(\"/kaggle/working/\"):\n    shutil.move(filename, \"/kaggle/working/stage1_train/\") \n\n!mkdir /kaggle/working/stage1_test/\n!unzip ../input/data-science-bowl-2018/stage1_test.zip\nfor filename in os.listdir(\"/kaggle/working/\"):\n    if filename != \"stage1_train\":\n        shutil.move(filename, \"/kaggle/working/stage1_test/\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = next(os.walk(TRAINING_PATH))[1]\ntest_ids = next(os.walk(TEST_PATH))[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = [item for item in train_ids if 'checkpoints' not in item]\n#indices = [i for i, s in enumerate(train_ids) if 'checkpoints' in s]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = np.zeros((len(train_ids), IMG_WIDTH, IMG_HEIGHT, IMG_CHANNELS),dtype=np.uint8)\nY_train = np.zeros((len(train_ids), IMG_WIDTH, IMG_HEIGHT, 1),dtype=np.bool)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Image Preparation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for n, id_ in tqdm(enumerate(train_ids), total = len(train_ids)):\n    path = TRAINING_PATH + id_\n    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_train[n] = img\n    mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype = np.bool)\n    for mask_file in next(os.walk(path+ '/masks/'))[2]:\n        mask_ = imread(path + '/masks/' + mask_file)\n        mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode = 'constant',\n                                     preserve_range = True), axis = -1 )\n        mask = np.maximum(mask, mask_)\nY_train[n] = mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_x = np.random.randint(0, len(train_ids))\nimshow(X_train[image_x])\nplt.show()\nimshow(np.squeeze(Y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nsizes_test = []\nprint('Resizing test images') \nfor n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n    path = TEST_PATH + id_\n    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    sizes_test.append([img.shape[0], img.shape[1]])\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_test[n] = img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Network construction","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"inputs = tf.keras.layers.Input((IMG_WIDTH, IMG_HEIGHT, IMG_CHANNELS))\ns = tf.keras.layers.Lambda(lambda x: x / 255)(inputs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Encoding part","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"c1 = tf.keras.layers.Conv2D(16, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(s)\nc1 = tf.keras.layers.Dropout(0.1)(c1)\nc1 = tf.keras.layers.Conv2D(16, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(c1)\np1 = tf.keras.layers.MaxPooling2D((2,2))(c1)\n\nc2 = tf.keras.layers.Conv2D(32, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(p1)\nc2 = tf.keras.layers.Dropout(0.1)(c2)\nc2 = tf.keras.layers.Conv2D(32, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(c2)\np2 = tf.keras.layers.MaxPooling2D((2,2))(c2)\n\nc3 = tf.keras.layers.Conv2D(64, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(p2)\nc3 = tf.keras.layers.Dropout(0.1)(c3)\nc3 = tf.keras.layers.Conv2D(64, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(c3)\np3 = tf.keras.layers.MaxPooling2D((2,2))(c3)\n\nc4 = tf.keras.layers.Conv2D(128, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(p3)\nc4 = tf.keras.layers.Dropout(0.1)(c4)\nc4 = tf.keras.layers.Conv2D(128, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(c4)\np4 = tf.keras.layers.MaxPooling2D((2,2))(c4)\n\nc5 = tf.keras.layers.Conv2D(256, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(p4)\nc5 = tf.keras.layers.Dropout(0.1)(c5)\nc5 = tf.keras.layers.Conv2D(256, (3,3), activation='relu', kernel_initializer = 'he_normal', padding = 'same')(c5)\np5 = tf.keras.layers.MaxPooling2D((2,2))(c5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Decoding part","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"u6 = tf.keras.layers.Conv2DTranspose(128, (2,2), strides = (2,2), padding='same')(c5)\nu6 = tf.keras.layers.concatenate([u6,c4])\nc6 = tf.keras.layers.Conv2D(128,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(u6)\nc6 = tf.keras.layers.Dropout(0.2)(c6)\nc6 = tf.keras.layers.Conv2D(128,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(c6)\n\nu7 = tf.keras.layers.Conv2DTranspose(64, (2,2), strides = (2,2), padding='same')(c6)\nu7 = tf.keras.layers.concatenate([u7,c3])\nc7 = tf.keras.layers.Conv2D(64,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(u7)\nc7 = tf.keras.layers.Dropout(0.2)(c7)\nc7 = tf.keras.layers.Conv2D(64,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(c7)\n\nu8 = tf.keras.layers.Conv2DTranspose(32, (2,2), strides = (2,2), padding='same')(c7)\nu8 = tf.keras.layers.concatenate([u8,c2])\nc8 = tf.keras.layers.Conv2D(32,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(u8)\nc8 = tf.keras.layers.Dropout(0.2)(c8)\nc8 = tf.keras.layers.Conv2D(32,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(c8)\n\nu9 = tf.keras.layers.Conv2DTranspose(16, (2,2), strides = (2,2), padding='same')(c8)\nu9 = tf.keras.layers.concatenate([u9,c1], axis=3)\nc9 = tf.keras.layers.Conv2D(16,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(u9)\nc9 = tf.keras.layers.Dropout(0.2)(c9)\nc9 = tf.keras.layers.Conv2D(16,(3,3),activation = 'relu', kernel_initializer = 'he_normal', padding = 'same')(c9)\n\noutputs = tf.keras.layers.Conv2D(1 , (1,1), activation = 'sigmoid')(c9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Model(inputs = [inputs], outputs = [outputs])\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpointer = tf.keras.callbacks.ModelCheckpoint('model_for_nuclei.h5', verbose=1, save_best_only = True)\ncallbacks = [tf.keras.callbacks.EarlyStopping(patience =2, monitor='val_loss'),tf.keras.callbacks.TensorBoard(log_dir='logs')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = model.fit(X_train,Y_train, validation_split = 0.1, batch_size=16, epochs=25, callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = np.random.randint(0, len(X_train))\n\n\npreds_train = model.predict(X_train[:int(X_train.shape[0]*0.9)], verbose=1)\npreds_val = model.predict(X_train[int(X_train.shape[0]*0.9):], verbose=1)\npreds_test = model.predict(X_test, verbose=1)\n\n \npreds_train_t = (preds_train > 0.5).astype(np.uint8)\npreds_val_t = (preds_val > 0.5).astype(np.uint8)\npreds_test_t = (preds_test > 0.5).astype(np.uint8)\n\n\n# Perform a sanity check on some random training samples\nix = np.random.randint(0, len(preds_train_t))\nimshow(X_train[ix])\nplt.show()\nimshow(np.squeeze(Y_train[ix]))\nplt.show()\nimshow(np.squeeze(preds_train_t[ix]))\nplt.show()\n\n# Perform a sanity check on some random validation samples\nix = np.random.randint(0, len(preds_val_t))\nimshow(X_train[int(X_train.shape[0]*0.9):][ix])\nplt.show()\nimshow(np.squeeze(Y_train[int(Y_train.shape[0]*0.9):][ix]))\nplt.show()\nimshow(np.squeeze(preds_val_t[ix]))\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}