{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip3 install tensorflow_addons","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:09:18.066745Z","iopub.execute_input":"2023-06-04T15:09:18.067096Z","iopub.status.idle":"2023-06-04T15:09:29.934626Z","shell.execute_reply.started":"2023-06-04T15:09:18.067069Z","shell.execute_reply":"2023-06-04T15:09:29.933504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import Sequential\nfrom tensorflow import keras\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport os\nfrom tensorflow.keras.datasets import mnist\nimport matplotlib.pyplot as plt\nfrom keras import backend as K\nimport seaborn as sns\nimport tensorflow_addons as tfa\nimport scipy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-05T10:29:50.291160Z","iopub.execute_input":"2023-06-05T10:29:50.291819Z","iopub.status.idle":"2023-06-05T10:29:50.297616Z","shell.execute_reply.started":"2023-06-05T10:29:50.291786Z","shell.execute_reply":"2023-06-05T10:29:50.296613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class modelAlexNet:\n    def __init__(self, activation = keras.activations.relu):\n        self.activation = activation\n        \n    def buildModel(self):\n        model = Sequential([\n            \n        keras.layers.Conv2D(filters=96, kernel_size=(3,3), strides=(4,4), activation=self.activation(), input_shape=(28, 28, 1)),\n        keras.layers.Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), activation=self.activation(), padding=\"same\"),\n        keras.layers.BatchNormalization(),\n        keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n        keras.layers.Conv2D(filters=128, kernel_size=(5,5), strides=(1,1), activation=self.activation(), padding=\"same\"),\n        keras.layers.BatchNormalization(),\n        keras.layers.Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), activation=self.activation(), padding=\"same\"),\n        keras.layers.BatchNormalization(),\n        keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation=self.activation(), padding=\"same\"),\n        keras.layers.BatchNormalization(),\n        keras.layers.Flatten(),\n        keras.layers.Dense(4096, activation=self.activation()),\n        keras.layers.Dense(4096, activation=self.activation()),\n        keras.layers.Dense(10, activation='softmax')\n                           \n                            ])\n        \n        return model\n        \n    def CompileAndSummarize(self, model):\n        model.compile(optimizer = tf.keras.optimizers.Adam(),\n                     loss = 'sparse_categorical_crossentropy',\n                     metrics = \"accuracy\")\n        model.summary()\n        ","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:40:27.040233Z","iopub.execute_input":"2023-06-05T10:40:27.040628Z","iopub.status.idle":"2023-06-05T10:40:27.053296Z","shell.execute_reply.started":"2023-06-05T10:40:27.040599Z","shell.execute_reply":"2023-06-05T10:40:27.052223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nLeakyReLU with Gaussian Noise (LeakyReLUGN)\n* LeakyReLU : **if x > 0 => x else if x <= 0 => 0.01*x**\n* Gaussian Noise : **alpha * gaussianpdf(mu, sigma)**\n* alpha, mu, sigma => trainable hyperparameters\n* LeakyReLUGN = **LeakyReLU + Gaussian Noise**\n* Apply Gaussian Smoothing to make the function more differentiable","metadata":{}},{"cell_type":"code","source":"class LeakyReLUGN(tf.keras.layers.Layer):\n    def __init__(self, alpha=1.0, sigma=1.0, mu=0.0):\n        super(LeakyReLUGN, self).__init__()\n        self.alpha = tf.Variable(alpha, trainable=True)\n        self.sigma = tf.Variable(sigma, trainable=True)\n        self.mu = tf.Variable(mu, trainable=True)\n    \n    def call(self, inputs):\n        leaky_relu = K.switch(inputs > 0, inputs, 0.01 * inputs)\n        gaussian = self.alpha * tf.exp(-tf.square(inputs - self.mu) / (2 * tf.square(self.sigma)))\n        ans = leaky_relu + gaussian\n        gaussian_smoothed_answer = tfa.image.gaussian_filter2d(ans, filter_shape = (3, 3), sigma = 1)\n        return gaussian_smoothed_answer","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:57:39.552576Z","iopub.execute_input":"2023-06-05T10:57:39.552937Z","iopub.status.idle":"2023-06-05T10:57:39.563439Z","shell.execute_reply.started":"2023-06-05T10:57:39.552907Z","shell.execute_reply":"2023-06-05T10:57:39.561900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alexnet = modelAlexNet(activation=LeakyReLUGN)\nmodel_lrgn = alexnet.buildModel()\nalexnet.CompileAndSummarize(model_lrgn)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:57:41.280329Z","iopub.execute_input":"2023-06-05T10:57:41.282767Z","iopub.status.idle":"2023-06-05T10:57:41.777785Z","shell.execute_reply.started":"2023-06-05T10:57:41.282736Z","shell.execute_reply":"2023-06-05T10:57:41.776498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train, y_train), (X_test, y_test) = mnist.load_data()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T08:48:44.394106Z","iopub.execute_input":"2023-06-05T08:48:44.394486Z","iopub.status.idle":"2023-06-05T08:48:44.647969Z","shell.execute_reply.started":"2023-06-05T08:48:44.394451Z","shell.execute_reply":"2023-06-05T08:48:44.647056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.astype('float32')\nX_test = X_test.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T08:48:44.649212Z","iopub.execute_input":"2023-06-05T08:48:44.650013Z","iopub.status.idle":"2023-06-05T08:48:44.708708Z","shell.execute_reply.started":"2023-06-05T08:48:44.649979Z","shell.execute_reply":"2023-06-05T08:48:44.707770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = np.expand_dims(X_train, -1)\nX_test = np.expand_dims(X_test, -1)\n\nX_train.shape, y_train.shape, X_test.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-05T08:48:44.710844Z","iopub.execute_input":"2023-06-05T08:48:44.711289Z","iopub.status.idle":"2023-06-05T08:48:44.718663Z","shell.execute_reply.started":"2023-06-05T08:48:44.711255Z","shell.execute_reply":"2023-06-05T08:48:44.717694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will look at which activation function overfits less using the following approach :\n* *Use EarlyStopping callback monitoring validation loss and compare which activation function trains for more epochs*","metadata":{}},{"cell_type":"code","source":"history_lrgn = model_lrgn.fit(X_train, y_train, batch_size = 8, epochs = 150, callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 15)], validation_data = (X_test, y_test))","metadata":{"execution":{"iopub.status.busy":"2023-06-05T08:48:49.167792Z","iopub.execute_input":"2023-06-05T08:48:49.168139Z","iopub.status.idle":"2023-06-05T08:49:24.014436Z","shell.execute_reply.started":"2023-06-05T08:48:49.168113Z","shell.execute_reply":"2023-06-05T08:49:24.012790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#track max - accuracy lrgn\n\nmax_lrgn_acc = max(history_lrgn.history['val_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.402572Z","iopub.status.idle":"2023-06-04T15:10:32.403263Z","shell.execute_reply.started":"2023-06-04T15:10:32.403031Z","shell.execute_reply":"2023-06-04T15:10:32.403052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(history_lrgn.history['accuracy'])\nsns.lineplot(history_lrgn.history['val_accuracy'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.404735Z","iopub.status.idle":"2023-06-04T15:10:32.405653Z","shell.execute_reply.started":"2023-06-04T15:10:32.405417Z","shell.execute_reply":"2023-06-04T15:10:32.405440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(history_lrgn.history['loss'])\nsns.lineplot(history_lrgn.history['val_loss'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.406898Z","iopub.status.idle":"2023-06-04T15:10:32.407870Z","shell.execute_reply.started":"2023-06-04T15:10:32.407599Z","shell.execute_reply":"2023-06-04T15:10:32.407625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alexnetr = modelAlexNet(activation=tf.keras.layers.ReLU)\nmodel_relu = alexnetr.buildModel()\nalexnetr.CompileAndSummarize(model_relu)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:33:54.777523Z","iopub.execute_input":"2023-06-04T15:33:54.777895Z","iopub.status.idle":"2023-06-04T15:33:54.932498Z","shell.execute_reply.started":"2023-06-04T15:33:54.777865Z","shell.execute_reply":"2023-06-04T15:33:54.931557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_relu = model_relu.fit(X_train, y_train, batch_size = 8, epochs = 150, callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 15)], validation_data = (X_test, y_test))","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:33:56.693313Z","iopub.execute_input":"2023-06-04T15:33:56.693674Z","iopub.status.idle":"2023-06-04T15:34:14.585140Z","shell.execute_reply.started":"2023-06-04T15:33:56.693637Z","shell.execute_reply":"2023-06-04T15:34:14.583684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#track max - accuracy relu\n\nmax_relu_acc = max(history_relu.history['val_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.413278Z","iopub.status.idle":"2023-06-04T15:10:32.414369Z","shell.execute_reply.started":"2023-06-04T15:10:32.414119Z","shell.execute_reply":"2023-06-04T15:10:32.414141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(history_relu.history['accuracy'])\nsns.lineplot(history_relu.history['val_accuracy'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.415553Z","iopub.status.idle":"2023-06-04T15:10:32.416451Z","shell.execute_reply.started":"2023-06-04T15:10:32.416207Z","shell.execute_reply":"2023-06-04T15:10:32.416234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(history_relu.history['loss'])\nsns.lineplot(history_relu.history['val_loss'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.417831Z","iopub.status.idle":"2023-06-04T15:10:32.418278Z","shell.execute_reply.started":"2023-06-04T15:10:32.418048Z","shell.execute_reply":"2023-06-04T15:10:32.418069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist([max_lrgn_acc, max_relu_acc])\nplt.xlabel('LeakyReLUGN vs ReLU')\nplt.ylabel('Max - accuracy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.419622Z","iopub.status.idle":"2023-06-04T15:10:32.420476Z","shell.execute_reply.started":"2023-06-04T15:10:32.420242Z","shell.execute_reply":"2023-06-04T15:10:32.420265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"activation_acc_ = {\n    'LeakyReLUGN': max_lrgn_acc,\n    'ReLU':max_relu_acc,\n}\n\nplt.bar(activation_acc_.keys(), activation_acc_.values(), width = 1.0, color='g')","metadata":{"execution":{"iopub.status.busy":"2023-06-04T15:10:32.421860Z","iopub.status.idle":"2023-06-04T15:10:32.422762Z","shell.execute_reply.started":"2023-06-04T15:10:32.422497Z","shell.execute_reply":"2023-06-04T15:10:32.422524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(activation_acc_)","metadata":{},"execution_count":null,"outputs":[]}]}