{"cells":[{"metadata":{"_uuid":"8ea8a9e596396179957c45cf6c402cb8216eacdd"},"cell_type":"markdown","source":"Here we call os module to know all about data and folders in dataset. Os module will let you walk through the doors fo dataset foder"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport sys\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport random\nrandom.seed = 42\nnp.random.seed = 42\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_path = \"../input/stage1_train/\"\ntest_path = \"../input/stage1_test/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40b7a77d2b151c87e00d6588cbc1dd88f98da675"},"cell_type":"code","source":"train_ids = next(os.walk(train_path))[1]\ntest_ids = next(os.walk(test_path))[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53b660f5e9e4a20e4da25711d4c0b37dd893d4cc"},"cell_type":"code","source":"print(\"total subfolders in stage1_train\",len(train_ids))\nprint(\"total subfolders in stage1_test \",len(test_ids))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c473708f451ce91077f1fa5a72ec42bee51f6533"},"cell_type":"markdown","source":"its a opencv demo to show how images look here we have taken a random image and try to print 3 versions of it A)original image B)by using Gaussian Blr c) by appling Threshold"},{"metadata":{"trusted":true,"_uuid":"91af6d3bd7664cca2b2d6f9ddae7ba99ceef5f53"},"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\ndef openCVdemo(path):\n    img = cv2.imread(path,0)\n    # Otsu's thresholding after Gaussian filtering\n    blur = cv2.GaussianBlur(img,(5,5),0)\n    ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)\n    # Plot Here\n    plt.figure(figsize=(15,5))\n    images = [blur, 0, th3]\n    titles = ['Original Image (X_train)','Gaussian filtered Image (OpenCV)',\"Segmened Image (OpenCV)\"]\n    plt.subplot(1,3,1),plt.imshow(img,'gray')\n    plt.title(titles[0]), plt.xticks([]), plt.yticks([])\n    plt.subplot(1,3,2),plt.imshow(images[0],'gray')\n    plt.title(titles[1]), plt.xticks([]), plt.yticks([])\n    plt.subplot(1,3,3),plt.imshow(images[2],'gray')\n    plt.title(titles[2]), plt.xticks([]), plt.yticks([])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b0338911fef6dc24652890b6e4f687be4e5c2e2"},"cell_type":"code","source":"n = random.randint(0,len(train_ids))\npath = train_path+train_ids[n]+\"/images/\"+train_ids[n]+\".png\"\nopenCVdemo(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8e597c2e5a3e24a2c21723d4cbe0b4ef3159d0a"},"cell_type":"code","source":"IMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c61819a2ef529885ecd6b76c288d3ebf0e998f19"},"cell_type":"code","source":"X_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nY_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09d950ec829de91dd5b031f8bbe42ad00349b823"},"cell_type":"code","source":"from tqdm import tqdm\nfrom itertools import chain\nfrom skimage.io import imread, imshow\nfrom skimage.transform import resize\nfrom skimage.morphology import label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2321c57d19c65184bc6384c50942dd548f27a7f"},"cell_type":"code","source":"\nprint('Getting and resizing train images and masks ... ')\nsys.stdout.flush()\nfor n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):\n    path = train_path + id_\n    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_train[n] = img\n    mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n    for mask_file in next(os.walk(path + '/masks/'))[2]:\n        mask_ = imread(path + '/masks/' + mask_file)\n        mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant', \n                                      preserve_range=True), axis=-1)\n        mask = np.maximum(mask, mask_)\n    Y_train[n] = mask\n\n# Get and resize test images\nX_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nsizes_test = []\nprint('Getting and resizing test images ... ')\nsys.stdout.flush()\nfor n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n    path = test_path + id_\n    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n    sizes_test.append([img.shape[0], img.shape[1]])\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_test[n] = img\n\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18372d565786eb3ce4abd43a50db95d476bd6a4d"},"cell_type":"code","source":"def plotTrainData(a,b):\n    for i in range(5):\n        ix = random.randint(0, len(train_ids))\n        plt.subplot(1,2,1)\n        plt.title(\"X_train\")\n        imshow(a[ix])\n        plt.axis('off')\n        plt.subplot(1,2,2)\n        plt.title(\"Y_train\")\n        imshow(np.squeeze(b[ix]))\n        plt.axis('off')\n        plt.show()\nplotTrainData(X_train,Y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6f7498b0a173d7d75a76d86a1f156723c4169d7"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.2)\nprint('\\nx_train',x_train.shape)\nprint('x_test',x_test.shape)\nprint('y_train',y_train.shape)\nprint('y_test',y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c02ff0bdbcd131789649ed93fe074dfa1a163630"},"cell_type":"code","source":"import tensorflow as tf\n# Define IoU metric\ndef mean_iou(y_true, y_pred):\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        y_pred_ = tf.to_int32(y_pred > t)\n        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)\n        K.get_session().run(tf.local_variables_initializer())\n        with tf.control_dependencies([up_opt]):\n            score = tf.identity(score)\n        prec.append(score)\n    return K.mean(K.stack(prec), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c95387808f4b780d2975f03eb4ea1688fac1a9a0"},"cell_type":"code","source":"from keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers.core import Dropout, Lambda\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"348b0cbb195011c3e31253e4aec23c378c87e48d"},"cell_type":"code","source":"# Build U-Net model\ninputs = Input((IMG_WIDTH , IMG_HEIGHT , IMG_CHANNELS))\ns = Lambda(lambda x:x/255)(inputs)\n#Layer1\nc1 = Conv2D(16,(3,3),activation = \"elu\",kernel_initializer = \"he_normal\",padding=\"same\")(s)\nc1 = Dropout(0.1)(c1)\nc1 = Conv2D(16,(3,3),activation = \"elu\",kernel_initializer = \"he_normal\", padding=\"same\")(c1)\np1 = MaxPooling2D((2,2))(c1)\n\n#Layer2\nc2 = Conv2D(32,(3,3),activation = \"elu\",kernel_initializer = \"he_normal\",padding=\"same\")(p1)\nc2 = Dropout(0.1)(c2)\nc2 = Conv2D(32,(3,3),activation = \"elu\",kernel_initializer = \"he_normal\", padding=\"same\")(c2)\np2 = MaxPooling2D((2,2))(c2)\n\n#Layer3\nc3 = Conv2D(64,(3,3),activation = \"elu\",kernel_initializer = \"he_normal\",padding=\"same\")(p2)\nc3 = Dropout(0.2)(c3)\nc3 = Conv2D(64,(3,3),activation = \"elu\",kernel_initializer = \"he_normal\", padding=\"same\")(c3)\np3 = MaxPooling2D((2,2))(c3)\n\n#layer4\nc4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p3)\nc4 = Dropout(0.2) (c4)\nc4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c4)\np4 = MaxPooling2D(pool_size=(2, 2)) (c4)\n\n#layer5\nc5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p4)\nc5 = Dropout(0.3) (c5)\nc5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c5)\n\nu6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same') (c5)\nu6 = concatenate([u6, c4])\nc6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u6)\nc6 = Dropout(0.2) (c6)\nc6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c6)\n\nu7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c6)\nu7 = concatenate([u7, c3])\nc7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u7)\nc7 = Dropout(0.2) (c7)\nc7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c7)\n\nu8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c7)\nu8 = concatenate([u8, c2])\nc8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u8)\nc8 = Dropout(0.1) (c8)\nc8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c8)\n\nu9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c8)\nu9 = concatenate([u9, c1], axis=3)\nc9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u9)\nc9 = Dropout(0.1) (c9)\nc9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c9)\n\noutputs = Conv2D(1, (1, 1), activation='sigmoid') (c9)\nmodel = Model(inputs=[inputs], outputs=[outputs])\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=[mean_iou])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"affc07858e51348615a1cece7017c22610e35bb0"},"cell_type":"code","source":"# Fit model\nearlystopper = EarlyStopping(patience=5, verbose=1)\ncheckpointer = ModelCheckpoint('model-dsbowl2018-1.h5', verbose=1, save_best_only=True)\nresults = model.fit(X_train, Y_train, validation_split=0.1, batch_size=16, epochs=50, \n                    callbacks=[earlystopper, checkpointer])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf5b75b219de01d398123b6d2ad30d04e35685ac"},"cell_type":"code","source":"# Predict on train, val and test\nmodel = load_model('model-dsbowl2018-1.h5', custom_objects={'mean_iou': mean_iou})\npreds_train = model.predict(X_train[:int(X_train.shape[0]*0.9)], verbose=1)\npreds_val = model.predict(X_train[int(X_train.shape[0]*0.9):], verbose=1)\npreds_test = model.predict(X_test, verbose=1)\n\n# Threshold predictions\npreds_train_t = (preds_train > 0.5).astype(np.uint8)\npreds_val_t = (preds_val > 0.5).astype(np.uint8)\npreds_test_t = (preds_test > 0.5).astype(np.uint8)\n\n# Create list of upsampled test masks\npreds_test_upsampled = []\nfor i in range(len(preds_test)):\n    preds_test_upsampled.append(resize(np.squeeze(preds_test[i]), \n                                       (sizes_test[i][0], sizes_test[i][1]), \n                                       mode='constant', preserve_range=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"797d37521eb7119aa6842548bc671b519f6af637"},"cell_type":"code","source":"# Perform a sanity check on some random training samples\nix = random.randint(0, len(preds_train_t))\nimshow(X_train[ix])\nplt.show()\nimshow(np.squeeze(Y_train[ix]))\nplt.show()\nimshow(np.squeeze(preds_train_t[ix]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc99c41541a7d257fd316ded6b086a1b7f8c187a"},"cell_type":"code","source":"# Perform a sanity check on some random validation samples\nix = random.randint(0, len(preds_val_t))\nimshow(X_train[int(X_train.shape[0]*0.9):][ix])\nplt.show()\nimshow(np.squeeze(Y_train[int(Y_train.shape[0]*0.9):][ix]))\nplt.show()\nimshow(np.squeeze(preds_val_t[ix]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ea920ca05e35775f6171b9d3b94778fc439ef05"},"cell_type":"code","source":"# Run-length encoding stolen from https://www.kaggle.com/rakhlin/fast-run-length-encoding-python\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cutoff=0.5):\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        yield rle_encoding(lab_img == i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9ddc7174bf0fc924b4a78b4594b0b216dcc57fd"},"cell_type":"code","source":"new_test_ids = []\nrles = []\nfor n, id_ in enumerate(test_ids):\n    rle = list(prob_to_rles(preds_test_upsampled[n]))\n    rles.extend(rle)\n    new_test_ids.extend([id_] * len(rle))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9869f94e66037299a5de4b328b04441fb87a34ca"},"cell_type":"code","source":"# Create submission DataFrame\nsub = pd.DataFrame()\nsub['ImageId'] = new_test_ids\nsub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\nsub.to_csv('sub-dsbowl2018-1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46ef8fdfb39237e58337373d37845f5da8848f03"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}