{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LeakyReLUGN : A Novel Activation Function to reduce Overfitting in Computer Vision Tasks","metadata":{}},{"cell_type":"markdown","source":"**Task :**\n - UNets : Cell Nuclei Segmentation\n \n\n*Compared with respect to ReLU*","metadata":{}},{"cell_type":"code","source":"!pip3 install tensorflow_addons","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Conv2DTranspose, concatenate\nfrom tensorflow.keras.models import Model\nfrom keras import metrics\nimport keras\nimport os\nimport random\nfrom tqdm import tqdm \nfrom zipfile import ZipFile\nfrom skimage.io import imread, imshow\nfrom skimage.transform import resize\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nfrom keras import backend as K\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2023-06-04T12:20:02.344437Z","iopub.execute_input":"2023-06-04T12:20:02.344790Z","iopub.status.idle":"2023-06-04T12:20:12.450070Z","shell.execute_reply.started":"2023-06-04T12:20:02.344762Z","shell.execute_reply":"2023-06-04T12:20:12.449138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Implement the U-Net Class\n\nclass UNet:\n    \n    def __init__(self, input_shape_,  activation_ = \"relu\"):\n        self.activation = activation_\n        self.input_shape = input_shape_\n    \n    def dice_coef(self, y_true, y_pred, smooth=100):\n        y_true = tf.cast(y_true, tf.float32)\n        y_pred = tf.cast(y_pred, tf.float32)\n        \n        y_true_f = K.flatten(y_true)\n        y_pred_f = K.flatten(y_pred)\n        intersection = K.sum(y_true_f * y_pred_f)\n        dice = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n        return dice\n    \n    def dice_coef_loss(self, y_true, y_pred):\n        return -self.dice_coef(y_true, y_pred)\n    \n    def iou(self, y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        y_pred = tf.cast(y_pred, tf.float32)\n\n        def f(y_true, y_pred):\n            intersection = (y_true * y_pred).sum()\n            union = y_true.sum() + y_pred.sum() - intersection\n            x = (intersection + 1e-15) / (union + 1e-15)\n            x = x.astype(np.float32)\n            return x\n        \n        return tf.numpy_function(f, [y_true, y_pred], tf.float32)\n\n    def buildModel(self):\n        \n        kernel_initializer =  'he_uniform'\n         \n        inputs = tf.keras.Input(shape = self.input_shape)\n        \n        # Block - 1\n    \n        s = inputs\n\n        #Contraction path\n        c1 = Conv2D(16, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(s)\n        c1 = Conv2D(16, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c1)\n        p1 = MaxPooling2D((2, 2))(c1)\n\n        c2 = Conv2D(32, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(p1)\n        c2 = Conv2D(32, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c2)\n        p2 = MaxPooling2D((2, 2))(c2)\n\n        c3 = Conv2D(64, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(p2)\n        c3 = Conv2D(64, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c3)\n        p3 = MaxPooling2D((2, 2))(c3)\n\n        c4 = Conv2D(128, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(p3)\n        c4 = Conv2D(128, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c4)\n        p4 = MaxPooling2D(pool_size=(2, 2))(c4)\n\n        c5 = Conv2D(256, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(p4)\n        c5 = Conv2D(256, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c5)\n\n        #Expansive path \n        u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n        u6 = concatenate([u6, c4])\n        c6 = Conv2D(128, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(u6)\n        c6 = Conv2D(128, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c6)\n\n        u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n        u7 = concatenate([u7, c3])\n        c7 = Conv2D(64, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(u7)\n        c7 = Conv2D(64, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c7)\n\n        u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n        u8 = concatenate([u8, c2])\n        c8 = Conv2D(32, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(u8)\n        c8 = Conv2D(32, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c8)\n\n        u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n        u9 = concatenate([u9, c1])\n        c9 = Conv2D(16, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(u9)\n        c9 = Conv2D(16, (3, 3), activation= self.activation, kernel_initializer=kernel_initializer, padding='same')(c9)\n\n        outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n\n        model = Model(inputs=[inputs], outputs=[outputs])\n    \n        model.summary()\n\n        return model\n \n    \n    def CompileandSummarize(self, model_):\n        model_.compile(optimizer = tf.keras.optimizers.Adam(), loss = self.dice_coef_loss, metrics = self.iou)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Preprocess training data\n\nwith ZipFile(\"../input/data-science-bowl-2018/stage1_train.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"./stage1_train\")\n    \nwith ZipFile(\"../input/data-science-bowl-2018/stage1_test.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"./stage1_test\")    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Define Constants\n\nseed = 42\nnp.random.seed = seed\n\nIMG_WIDTH = 96\nIMG_HEIGHT = 96\nIMG_CHANNELS = 3\n\nTRAIN_PATH = './stage1_train/'\nTEST_PATH = './stage1_test/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = next(os.walk(TRAIN_PATH))[1]\ntest_ids = next(os.walk(TEST_PATH))[1]\n\nX_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nY_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n\nprint('Resizing training images and masks')\nfor n, file_name in tqdm(enumerate(train_ids)):   \n    path = TRAIN_PATH + file_name\n    img = imread(path + '/images/' + file_name + '.png')[:,:,:IMG_CHANNELS]  \n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_train[n] = img  #Fill empty X_train with values from img\n    mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n    for mask_file in next(os.walk(path + '/masks/'))[2]:\n        mask_ = imread(path + '/masks/' + mask_file)\n        mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant',  \n                                      preserve_range=True), axis=-1)\n        mask = np.maximum(mask, mask_)  \n            \n    Y_train[n] = mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5% Validation Images (34 Images)\nX_test, y_test  = X_train[636:], Y_train[636:]\nX_train, Y_train = X_train[0: 636], Y_train[0:636]\n(X_train.shape, Y_train.shape), (X_test.shape, y_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_x = random.randint(0, 636)\nimshow(X_train[image_x])\nplt.show()\nimshow(np.squeeze(Y_train[image_x]))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LeakyReLU with Gaussian Noise (LeakyReLUGN)\n* LeakyReLU : **if x > 0 => x else if x <= 0 => 0.01*x**\n* Gaussian Noise : **alpha * gaussianpdf(mu, sigma)**\n* alpha, mu, sigma => trainable hyperparameters\n* LeakyReLUGN = **LeakyReLU + Gaussian Noise**\n* Final Step : Apply Gaussian Smoothing to make the function more differentiable","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport scipy.ndimage.filters as filters\nfrom tensorflow.keras import backend as K\n\nclass LeakyReLUGN(tf.keras.layers.Layer):\n    def __init__(self, alpha=1.0, sigma=1.0, mu=0.0):\n        super(LeakyReLUGN, self).__init__()\n        self.alpha = tf.Variable(alpha, trainable=True)\n        self.sigma = tf.Variable(sigma, trainable=True)\n        self.mu = tf.Variable(mu, trainable=True)\n\n    def call(self, inputs):\n        leaky_relu = K.switch(inputs > 0, inputs, 0.01 * inputs)\n        gaussian = self.alpha * tf.exp(-tf.square(inputs - self.mu) / (2 * tf.square(self.sigma)))\n        ans = leaky_relu + gaussian\n        gaussian_smoothed_answer = tfa.image.gaussian_filter2d(ans, filter_shape = (3, 3), sigma = 1)\n        return gaussian_smoothed_answer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will look at which activation function overfits less using the following approach :\n* *Use EarlyStopping callback monitoring validation loss and compare which activation function trains for more epochs*","metadata":{}},{"cell_type":"code","source":"# model training LeakyReLUGN\n\n\nUNet_LeakyReLUGN = UNet((96, 96, 3), activation_=LeakyReLUGN())\nmodel_LeakyReLUGN = UNet_LeakyReLUGN.buildModel()\nUNet_LeakyReLUGN.CompileandSummarize(model_LeakyReLUGN)\nresults_LeakyReLUGN = model_LeakyReLUGN.fit(x = X_train, y = Y_train, batch_size = 8, epochs=150, callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 10)],  validation_data = (X_test, y_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(results_LeakyReLUGN.history['iou'])\nsns.lineplot(results_LeakyReLUGN.history['val_iou'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(results_LeakyReLUGN.history['loss'])\nsns.lineplot(results_LeakyReLUGN.history['val_loss'])\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model training relu\n\n\n\nUNet_ReLU = UNet((96, 96, 3), activation_=\"relu\")\nmodel_ReLU = UNet_ReLU.buildModel()\nUNet_ReLU.CompileandSummarize(model_ReLU)\nresults_ReLU = model_ReLU.fit(x = X_train, y = Y_train, batch_size = 8, epochs=150, callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 10)],  validation_data = (X_test, y_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(results_ReLU.history['iou'])\nsns.lineplot(results_ReLU.history['val_iou'])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(results_ReLU.history['loss'])\nsns.lineplot(results_ReLU.history['val_loss'])\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}