{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import zipfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists('/kaggle/data/') :\n    os.makedirs('/kaggle/data/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with zipfile.ZipFile('/kaggle/input/data-science-bowl-2018/stage1_train.zip', 'r') as existing_zip:\n    existing_zip.extractall('/kaggle/data/train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with zipfile.ZipFile('/kaggle/input/data-science-bowl-2018/stage1_test.zip', 'r') as existing_zip:\n    existing_zip.extractall('/kaggle/data/test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with zipfile.ZipFile('/kaggle/input/data-science-bowl-2018/stage1_solution.csv.zip', 'r') as existing_zip:\n    existing_zip.extractall('/kaggle/data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with zipfile.ZipFile('/kaggle/input/data-science-bowl-2018/stage1_train_labels.csv.zip', 'r') as existing_zip:\n    existing_zip.extractall('/kaggle/data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/data'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = next(os.walk('/kaggle/data/train'))[1]\ntest_ids = next(os.walk('/kaggle/data/test'))[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage.io import imread, imshow, imread_collection, concatenate_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATH='/kaggle/data/train/'\nTEST_PATH='/kaggle/data/test/'\nimg = imread(TRAIN_PATH+str(train_ids[1])+'/images/'+str(train_ids[1])+'.png')[:,:,:IMG_CHANNELS]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img=[]\nfor i,id_ in enumerate(next(os.walk(TRAIN_PATH+str(train_ids[1])+'/masks/'))[2]):\n    img.append(imread(TRAIN_PATH+str(train_ids[1])+'/masks/'+str(id_)))\nimg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(img[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(img[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(img[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(img[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nY_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage.transform import resize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for n,id_ in tqdm(enumerate(train_ids),total=len(train_ids)):\n    img=imread(TRAIN_PATH+id_+'/images/'+id_+'.png')\n    img=resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)[:,:,:IMG_CHANNELS]\n    X_train[n]=img\n    mask=np.zeros((IMG_HEIGHT,IMG_WIDTH,1),dtype=np.bool)\n    for maskname in next(os.walk(TRAIN_PATH+id_+'/masks/'))[2]:\n        mask_=imread(TRAIN_PATH+id_+'/masks/'+maskname)\n        mask_=np.expand_dims(resize(mask_,(IMG_HEIGHT,IMG_WIDTH),mode='constant', preserve_range=True),axis=-1)\n        mask=np.maximum(mask,mask_)\n    Y_train[n] = mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nsizes_test = []\n\nfor n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n    img=imread(TEST_PATH +id_+ '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    sizes_test.append([img.shape[0], img.shape[1]])\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_test[n] = img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nix = random.randint(0, len(train_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(X_train[ix])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imshow(np.squeeze(Y_train[ix]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keras.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mean_iou(y_true, y_pred):\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        y_pred_ = tf.cast(y_pred > t, tf.int32)\n        score, up_opt = tf.compat.v1.metrics.mean_iou(y_true, y_pred_, 2)\n        K.get_session().run(tf.local_variables_initializer())\n        with tf.control_dependencies([up_opt]):\n            score = tf.identity(score)\n        prec.append(score)\n    return K.mean(K.stack(prec), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\ns = Lambda(lambda x: x / 255) (inputs)\n\nc1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (s)\nc1 = Dropout(0.1) (c1)\nc1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c1)\np1 = MaxPooling2D((2, 2)) (c1)\n\nc2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p1)\nc2 = Dropout(0.1) (c2)\nc2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c2)\np2 = MaxPooling2D((2, 2)) (c2)\n\nc3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p2)\nc3 = Dropout(0.2) (c3)\nc3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c3)\np3 = MaxPooling2D((2, 2)) (c3)\n\nc4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p3)\nc4 = Dropout(0.2) (c4)\nc4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c4)\np4 = MaxPooling2D(pool_size=(2, 2)) (c4)\n\nc5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p4)\nc5 = Dropout(0.3) (c5)\nc5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c5)\n\nu6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same') (c5)\nu6 = concatenate([u6, c4])\nc6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u6)\nc6 = Dropout(0.2) (c6)\nc6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c6)\n\nu7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c6)\nu7 = concatenate([u7, c3])\nc7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u7)\nc7 = Dropout(0.2) (c7)\nc7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c7)\n\nu8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c7)\nu8 = concatenate([u8, c2])\nc8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u8)\nc8 = Dropout(0.1) (c8)\nc8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c8)\n\nu9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c8)\nu9 = concatenate([u9, c1], axis=3)\nc9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u9)\nc9 = Dropout(0.1) (c9)\nc9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c9)\n\noutputs = Conv2D(1, (1, 1), activation='sigmoid') (c9)\n\nmodel = Model(inputs=[inputs], outputs=[outputs])\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"earlystopper = EarlyStopping(patience=5, verbose=1)\ncheckpointer = ModelCheckpoint('model-dsbowl2018.h5', verbose=1, save_best_only=True)\n\nresults = model.fit(X_train, Y_train, validation_split=0.1, batch_size=16, epochs=50, \n                    callbacks=[earlystopper, checkpointer])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = load_model('model-dsbowl2018.h5', custom_objects={'mean_iou': mean_iou})\npreds_test = model.predict(X_test, verbose=1)\n\npreds_test_upsampled = []\nfor i in range(len(preds_test)):\n    preds_test_upsampled.append(resize(np.squeeze(preds_test[i]), \n                                       (sizes_test[i][0], sizes_test[i][1]), \n                                       mode='constant', preserve_range=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage.morphology import label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cutoff=0.5):\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        yield rle_encoding(lab_img == i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_test_ids = []\nrles = []\nfor n, id_ in enumerate(test_ids):\n    rle = list(prob_to_rles(preds_test_upsampled[n]))\n    rles.extend(rle)\n    new_test_ids.extend([id_] * len(rle))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['ImageId'] = new_test_ids\nsub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\nsub.to_csv('sub-dsbowl2018.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}