{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from zipfile import ZipFile \nimport os\nimport cv2 \nimport numpy as np\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#unzip train data\nfile_name = \"../input/data-science-bowl-2018/stage1_train.zip\" \nwith ZipFile(file_name, 'r') as zip: \n    print('Extracting the files') \n    zip.extractall(\"traindata\") \n    print('Done!')\n\n#unzip test data\nfile_name = \"../input/data-science-bowl-2018/stage1_test.zip\" \nwith ZipFile(file_name, 'r') as zip: \n    print('Extracting the files') \n    zip.extractall(\"testdata\") \n    print('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trian path\ntrainPath=\"./traindata\"\nfolder=os.listdir(trainPath)\nprint(folder[:5])\nprint()\n#fil for patient id file\npathfile1=os.path.join(trainPath,folder[5])\nfile1=os.listdir(pathfile1)\nprint(file1)\n\n#image for patient \nimagePath=os.path.join(pathfile1,file1[1])\nimages=os.listdir(imagePath)\nprint(images)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#read and show image\nreadimage=plt.imread(os.path.join(imagePath,images[0]))\n\nfrom skimage.color import rgb2gray\n\nplt.imshow(rgb2gray(readimage))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow((readimage))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#show mask\n\n#mask for patient \nmaskPath=os.path.join(pathfile1,file1[0])\nmasks=os.listdir(maskPath)\nprint(\"the number of masks\",len(masks))\n\n\nplt.figure(figsize=(50,50))\nfor i in range(len(masks)):\n    plt.subplot(15,10,i+1)\n    plt.imshow(plt.imread(os.path.join(maskPath,masks[i])),cmap=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#to Comline the masks in individual image\nmask = np.zeros((128, 128, 1), dtype=np.bool)\nfor i in range(len(masks)):\n\n            mask_ = plt.imread(os.path.join(maskPath,masks[i]))\n            mask_ = np.expand_dims(cv2.resize(mask_, (128, 128)), axis=-1)\n    #         print(mask_)\n            mask = np.maximum(mask, mask_)\nplt.imshow(np.squeeze(mask))\nplt.title(np.squeeze(mask).shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pathlib\nimport imageio\nimport numpy as np\n\ntraining_images = pathlib.Path(\"./traindata\").glob('*/images/*.png')\ntraining_images = [x for x in training_images]\nprint(training_images)\n\nimagePath = training_images[10]\nimage = plt.imread(imagePath)\nprint(\"image shape \",image.shape )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read train data\nX_Train=[]\nY_Train=[]\n\n\n\nfor patient in range(len(folder)):\n    #fil for patient id file\n    pathfile=os.path.join(trainPath,folder[patient])\n    file=os.listdir(pathfile)\n\n    #image for patient \n    imagePath=os.path.join(pathfile,file[1])\n    images=os.listdir(imagePath)\n    #print(images)\n\n    #read and show image\n    image=plt.imread(os.path.join(imagePath,images[0]))[:,:,:3]\n    image=cv2.resize(image,(128,128))\n    print(image.shape)\n\n    X_Train.append(np.array(image))\n    \n    #mask for patient \n    maskPath=os.path.join(pathfile,file[0])\n    masks=os.listdir(maskPath)\n    print(\"the number of masks\",len(masks))\n    \n\n    #to Comline the masks in individual image\n    mask = np.zeros((128, 128, 1), dtype=np.bool)\n    for i in range(len(masks)):\n\n            mask_ =plt.imread(os.path.join(maskPath,masks[i]))\n            mask_ = np.expand_dims(cv2.resize(mask_, (128, 128)), axis=-1)\n    #         print(mask_)\n            mask = np.maximum(mask, mask_)\n    Y_Train.append(np.array(mask))\n\n        \n        \n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_Train=np.array(X_Train)\nY_Train=np.array(Y_Train)\nprint(X_Train.shape)\nprint(Y_Train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testPath=\"./testdata\"\ntestfolder=os.listdir(testPath)\nprint(testfolder)\n\npathfile=os.path.join(testPath,testfolder[0])\nfile=os.listdir(pathfile)\nprint(file)\n\n# #image for patient \n# imagePath=os.path.join(pathfile1,file1[1])\n# images=os.listdir(imagePath)\n# print(images)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nX_Test=[]\n\ntestPath=\"./testdata\"\ntestfolder=os.listdir(testPath)\nfor patient in range(len(testfolder)):\n    #fil for patient id file\n    pathfile=os.path.join(testPath,testfolder[patient])\n    file=os.listdir(pathfile)\n\n    #image for patient \n    imagePath=os.path.join(pathfile,file[0])\n    images=os.listdir(imagePath)\n#     print(images)\n\n    #read and show image\n    image1=plt.imread(os.path.join(imagePath,images[0]))[:,:,:3]\n    image1=cv2.resize(image1,(128,128))\n    print(image1.shape)\n    X_Test.append(np.array(image1))\n    \n  \n        \n        \n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_Test=np.array(X_Test)\nfor i in range(len(X_Test)):\n    X_Test[i]=cv2.resize(X_Test[i],(128,128))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n        plt.figure(figsize=(10,10))\n        plt.subplot(1,2,1)\n        plt.title(\"Real\")\n        plt.imshow(X_Train[i])\n        plt.subplot(1,2,2)\n        plt.title(\"Mask\")\n        plt.imshow(np.squeeze(Y_Train[i]))\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# bulid U-net Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#donot forget to add DropOut\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D ,Flatten ,Dense ,Concatenate ,UpSampling2D,Conv2DTranspose\nfrom tensorflow.keras import Sequential\n#Build model\ninputs=tf.keras.layers.Input(shape=(128,128,3))\ns=tf.keras.layers.Lambda(lambda x:x/255)(inputs)\n\n#Contraction\nc1=Conv2D(16, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(s) # 16*(3*3)*3(channel)+16\nc1=Conv2D(16, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(c1) #16 *(3*3)*16 +16\nprint(\"first convolutional layer\",c1.shape)\np1=MaxPooling2D((2,2),strides=2)(c1)\nprint(\"first maxPool layer\",p1.shape,\"\\n\")\n\nc2=Conv2D(32, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(p1)#16*(3*3)*32+32\nc2=Conv2D(32, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c2)\nprint(\"second convolutional layer\",c2.shape)\np2=MaxPooling2D((2,2),strides=2)(c2)\nprint(\"second maxPool layer\",p2.shape,\"\\n\")\n\nc3=Conv2D(64, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(p2)\nc3=Conv2D(64, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c3)\nprint(\"third convolutional layer\",c3.shape)\np3=MaxPooling2D((2,2),strides=2)(c3)\nprint(\"third maxPool layer\",p3.shape,\"\\n\")\n\nc4=Conv2D(128, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(p3)\nc4=Conv2D(128, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c4)\nprint(\"fourth convolutional layer\",c4.shape)\np4=MaxPooling2D((2,2),strides=2)(c4)\nprint(\"fourth  maxPool layer\",p4.shape,\"\\n\")\n\nc5=Conv2D(256, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(p4)\nc5=Conv2D(256, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c5)\nprint(\"fifth convolutional layer\",c5.shape)\n\n#Expansive path\n\nU6=Conv2DTranspose(128, kernel_size=(2,2),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\",strides=(2,2))(c5)\nprint(\"\\nfirst upSampling\",U6.shape)\nU6=Concatenate()([U6, c4])\nprint(\"first Concatenate\",U6.shape,\"\\n\")\nc6=Conv2D(128, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(U6)\nc6=Conv2D(128, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c6)\nprint(\"sixth convolutional layer\",c6.shape)\n\n\nU7=Conv2DTranspose(64, kernel_size=(2,2),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\",strides=(2,2))(c6)\nprint(\"\\nsecond upSampling\",U7.shape)\nU7=Concatenate()([U7, c3])\nprint(\"second Concatenate\",U6.shape)\nc7=Conv2D(64, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(U7)\nc7=Conv2D(64, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c7)\nprint(\"\\nseventh  convolutional layer\",c7.shape)\n\n\nU8=Conv2DTranspose(32, kernel_size=(2,2),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\",strides=(2,2))(c7)\nprint(\"\\nthird upSampling\",U8.shape)\nU8=Concatenate()([U8, c2])\nprint(\"third Concatenate\",U8.shape)\nc8=Conv2D(32, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(U8)\nc8=Conv2D(32, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c8)\nprint(\"\\nEIGTH  convolutional layer\",c8.shape)\n\n\n\nU9=Conv2DTranspose(16, kernel_size=(2,2),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\",strides=(2,2))(c8)\nprint(\"\\nfourth upSampling\",U9.shape)\nU9=Concatenate()([U9, c1])\nprint(\"fourth Concatenate\",U9.shape)\nc9=Conv2D(16, kernel_size=(3,3),padding=\"same\", kernel_initializer='he_normal',activation=\"relu\")(U9)\nc9=Conv2D(16, kernel_size=(3,3),activation=\"relu\", kernel_initializer='he_normal',padding=\"same\")(c9)\nprint(\"\\nninth  convolutional layer\",c9.shape)\n\noutput=Conv2D(1, kernel_size=(1,1),activation=\"sigmoid\")(c9)\nprint(\"\\nOutput layer\",output.shape)\n\nmodel=tf.keras.Model(inputs=[inputs],outputs=[output])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel.compile(optimizer=\"adam\",loss=\"binary_crossentropy\",metrics=[\"accuracy\"])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checkpoint\ncheckpoint=tf.keras.callbacks.ModelCheckpoint(\"modelForNuclei.h5\", verbose=1, save_best_only=True)\n#Callbacks\n\nCallbacks=[\n    tf.keras.callbacks.EarlyStopping(patience=5, monitor=\"val_loss\"),\n    tf.keras.callbacks.TensorBoard(log_dir='log'),\n    checkpoint\n    \n]\n\n\n#Fit model \nResults=model.fit(X_Train,Y_Train,validation_split=0.01, batch_size=4 , epochs=100 , callbacks=Callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import load_model\nload_model(\"./modelForNuclei.h5\").evaluate(X_Train,Y_Train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_train = model.predict(X_Train[9:10], verbose=1)\npredict=cv2.resize(np.squeeze(preds_train), (128,128))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\n\nplt.subplot(1,3,1)\nplt.title(\"Predict\")\nplt.imshow(predict)\n\n\nplt.subplot(1,3,2)\nplt.title(\"ground truth\")\nplt.imshow(Y_Train[9].reshape(128,128))\n\n\nplt.subplot(1,3,3)\nplt.title(\"real\")\nplt.imshow(X_Train[9])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\nfor i in range(1,10):\n    preds_test = model.predict(X_Test[i-1:i], verbose=1)\n    predict=cv2.resize(np.squeeze(preds_test), (128,128))\n    plt.figure(figsize=(10,10))\n    plt.subplot(1,2,1)\n    plt.title(\"Predict\")\n    plt.imshow(predict)\n\n\n    plt.subplot(1,2,2)\n    plt.title(\"real\")\n    plt.imshow(X_Test[i-1])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}