{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-26T19:59:11.780051Z","iopub.execute_input":"2022-03-26T19:59:11.780723Z","iopub.status.idle":"2022-03-26T19:59:11.793442Z","shell.execute_reply.started":"2022-03-26T19:59:11.780686Z","shell.execute_reply":"2022-03-26T19:59:11.792377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing libraries\nfrom sklearn.model_selection import train_test_split \n\n\nimport tensorflow as tf\n\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.initializers import RandomNormal\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport time\n\nfrom tensorflow.keras.models import Sequential \nfrom tensorflow.keras.layers import Dense, Activation \nfrom tensorflow.keras.layers import BatchNormalization\nfrom tensorflow.keras.layers import Dropout\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:11.829014Z","iopub.execute_input":"2022-03-26T19:59:11.829432Z","iopub.status.idle":"2022-03-26T19:59:11.835333Z","shell.execute_reply.started":"2022-03-26T19:59:11.829400Z","shell.execute_reply":"2022-03-26T19:59:11.834401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if keras is not using tensorflow as backend, we can set \"KERAS_BACKEND=tensorflow\" \n!set \"KERAS_BACKEND=tensorflow\"","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:11.854071Z","iopub.execute_input":"2022-03-26T19:59:11.854351Z","iopub.status.idle":"2022-03-26T19:59:12.573363Z","shell.execute_reply.started":"2022-03-26T19:59:11.854323Z","shell.execute_reply":"2022-03-26T19:59:12.572256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Importing dataset","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:12.575588Z","iopub.execute_input":"2022-03-26T19:59:12.575828Z","iopub.status.idle":"2022-03-26T19:59:16.328971Z","shell.execute_reply.started":"2022-03-26T19:59:12.575781Z","shell.execute_reply":"2022-03-26T19:59:16.328204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape)\nprint(\"Number of training examples :\", train_data.shape[0], \"and each image is of shape (%d)\"%(train_data.shape[1]))\nprint(\"Number of training examples :\", test_data.shape[0], \"and each image is of shape (%d)\"%(test_data.shape[1]))\nprint(test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:16.330345Z","iopub.execute_input":"2022-03-26T19:59:16.330586Z","iopub.status.idle":"2022-03-26T19:59:16.339454Z","shell.execute_reply.started":"2022-03-26T19:59:16.330552Z","shell.execute_reply":"2022-03-26T19:59:16.338714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# An example data point\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:16.342000Z","iopub.execute_input":"2022-03-26T19:59:16.342336Z","iopub.status.idle":"2022-03-26T19:59:16.360632Z","shell.execute_reply.started":"2022-03-26T19:59:16.342303Z","shell.execute_reply":"2022-03-26T19:59:16.359841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:16.362223Z","iopub.execute_input":"2022-03-26T19:59:16.362566Z","iopub.status.idle":"2022-03-26T19:59:16.408141Z","shell.execute_reply.started":"2022-03-26T19:59:16.362508Z","shell.execute_reply":"2022-03-26T19:59:16.407450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.label.dtype","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:16.409287Z","iopub.execute_input":"2022-03-26T19:59:16.409512Z","iopub.status.idle":"2022-03-26T19:59:16.415911Z","shell.execute_reply.started":"2022-03-26T19:59:16.409482Z","shell.execute_reply":"2022-03-26T19:59:16.415170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:16.417537Z","iopub.execute_input":"2022-03-26T19:59:16.418134Z","iopub.status.idle":"2022-03-26T19:59:18.107377Z","shell.execute_reply.started":"2022-03-26T19:59:16.418099Z","shell.execute_reply":"2022-03-26T19:59:18.106425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:18.109003Z","iopub.execute_input":"2022-03-26T19:59:18.109263Z","iopub.status.idle":"2022-03-26T19:59:20.311431Z","shell.execute_reply.started":"2022-03-26T19:59:18.109227Z","shell.execute_reply":"2022-03-26T19:59:20.310693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Splitting train-data","metadata":{}},{"cell_type":"code","source":"# splitting data into dependent and independent variable\nX, y = train_data.drop('label', axis=1), train_data['label']\n\n# splitting data into train-validation set\nx_train, x_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.312682Z","iopub.execute_input":"2022-03-26T19:59:20.313177Z","iopub.status.idle":"2022-03-26T19:59:20.593666Z","shell.execute_reply.started":"2022-03-26T19:59:20.313127Z","shell.execute_reply":"2022-03-26T19:59:20.592840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we can observe that in the above dataframes' each cell is having a value between 0-255\n# Let us normalize the dataa\n# X => (X - Xmin)/(Xmax-Xmin) = X/255\n\nx_train = x_train/255\nx_val = x_val/255\ntest_data = test_data/255","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.597581Z","iopub.execute_input":"2022-03-26T19:59:20.597807Z","iopub.status.idle":"2022-03-26T19:59:20.786143Z","shell.execute_reply.started":"2022-03-26T19:59:20.597769Z","shell.execute_reply":"2022-03-26T19:59:20.785382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.787437Z","iopub.execute_input":"2022-03-26T19:59:20.787989Z","iopub.status.idle":"2022-03-26T19:59:20.794901Z","shell.execute_reply.started":"2022-03-26T19:59:20.787949Z","shell.execute_reply":"2022-03-26T19:59:20.794209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# here we are having a class number for each image\nprint(\"Class label of first image :\", y_train[:1])\n\n# we will convert the class labels into a 10 dimensional vector\n# ex: if an image is 5, we will convert it into 5 => [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]\n# this conversion is needed for MLPs \n\nY_train = utils.to_categorical(y_train, 10, dtype='int64') \nY_val = utils.to_categorical(y_val, 10, dtype='int64')\n\nprint(\"After converting the output into a vector : \",Y_train[:1])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.796247Z","iopub.execute_input":"2022-03-26T19:59:20.796934Z","iopub.status.idle":"2022-03-26T19:59:20.807590Z","shell.execute_reply.started":"2022-03-26T19:59:20.796897Z","shell.execute_reply":"2022-03-26T19:59:20.806670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.head()\n# they are matching with below Y_train","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.809085Z","iopub.execute_input":"2022-03-26T19:59:20.809384Z","iopub.status.idle":"2022-03-26T19:59:20.818494Z","shell.execute_reply.started":"2022-03-26T19:59:20.809351Z","shell.execute_reply":"2022-03-26T19:59:20.817784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(Y_train)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.819599Z","iopub.execute_input":"2022-03-26T19:59:20.820033Z","iopub.status.idle":"2022-03-26T19:59:20.828697Z","shell.execute_reply.started":"2022-03-26T19:59:20.819981Z","shell.execute_reply":"2022-03-26T19:59:20.828082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train[:5]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.830049Z","iopub.execute_input":"2022-03-26T19:59:20.830460Z","iopub.status.idle":"2022-03-26T19:59:20.838474Z","shell.execute_reply.started":"2022-03-26T19:59:20.830427Z","shell.execute_reply":"2022-03-26T19:59:20.837638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model performance Visualization","metadata":{}},{"cell_type":"code","source":"# this function will be used to update the plots for each epoch and error\ndef plt_dynamic(x, vy, ty, ax, colors=['b']):\n    ax.plot(x, vy, 'b', label=\"Validation Loss\")\n    ax.plot(x, ty, 'r', label=\"Train Loss\")\n    \n    plt.legend()\n    plt.grid()\n    fig.canvas.draw()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.841130Z","iopub.execute_input":"2022-03-26T19:59:20.841991Z","iopub.status.idle":"2022-03-26T19:59:20.848177Z","shell.execute_reply.started":"2022-03-26T19:59:20.841966Z","shell.execute_reply":"2022-03-26T19:59:20.847338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Model Building: \n* With only single Perceptron,  \n* Softmax activation function, \n* SGD optimizer","metadata":{}},{"cell_type":"code","source":"# model parameters\n\noutput_dim = 10\ninput_dim = x_train.shape[1]\nprint(input_dim)\n\nbatch_size = 128 \nepoch = 20","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.849364Z","iopub.execute_input":"2022-03-26T19:59:20.849821Z","iopub.status.idle":"2022-03-26T19:59:20.857227Z","shell.execute_reply.started":"2022-03-26T19:59:20.849765Z","shell.execute_reply":"2022-03-26T19:59:20.856371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(output_dim, input_dim=input_dim, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.858409Z","iopub.execute_input":"2022-03-26T19:59:20.858824Z","iopub.status.idle":"2022-03-26T19:59:20.876688Z","shell.execute_reply.started":"2022-03-26T19:59:20.858765Z","shell.execute_reply":"2022-03-26T19:59:20.876065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Compiling the model\n# Note: when using the categorical_crossentropy loss, our targets should be in categorical format \n# (e.g. if we have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros except \n# for a 1 at the index corresponding to the class of the sample).\n# that is why we converted out labels into vectors\n\nmodel.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.877687Z","iopub.execute_input":"2022-03-26T19:59:20.878304Z","iopub.status.idle":"2022-03-26T19:59:20.887885Z","shell.execute_reply.started":"2022-03-26T19:59:20.878268Z","shell.execute_reply":"2022-03-26T19:59:20.887177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fitting the model and recording the history\nhistory = model.fit(x_train, Y_train, steps_per_epoch=500, epochs=epoch, verbose=1, validation_data=(x_val, Y_val)) \n","metadata":{"execution":{"iopub.status.busy":"2022-03-26T19:59:20.888940Z","iopub.execute_input":"2022-03-26T19:59:20.889471Z","iopub.status.idle":"2022-03-26T20:00:02.609854Z","shell.execute_reply.started":"2022-03-26T19:59:20.889435Z","shell.execute_reply":"2022-03-26T20:00:02.609078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch'); \nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\n\nvy = history.history['val_loss']\nty = history.history['loss']\n\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:02.611653Z","iopub.execute_input":"2022-03-26T20:00:02.611912Z","iopub.status.idle":"2022-03-26T20:00:03.126750Z","shell.execute_reply.started":"2022-03-26T20:00:02.611878Z","shell.execute_reply":"2022-03-26T20:00:03.126057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Model Building: \n* With Multi Layer perceptron (MLP),  \n* sigmoid (and softmax) activation function, \n* SGD optimizer","metadata":{}},{"cell_type":"code","source":"# Multilayer perceptron\n\nmodel_sigmoid = Sequential()\nmodel_sigmoid.add(Dense(512, activation='sigmoid', input_shape=(input_dim,)))\nmodel_sigmoid.add(Dense(128, activation='sigmoid'))\nmodel_sigmoid.add(Dense(output_dim, activation='softmax'))\n\nmodel_sigmoid.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:03.128253Z","iopub.execute_input":"2022-03-26T20:00:03.128648Z","iopub.status.idle":"2022-03-26T20:00:03.160615Z","shell.execute_reply.started":"2022-03-26T20:00:03.128609Z","shell.execute_reply":"2022-03-26T20:00:03.159952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_sigmoid.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_sigmoid.fit(x_train, Y_train, batch_size=batch_size, epochs=epoch, verbose=1, validation_data=(x_val, Y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:03.162417Z","iopub.execute_input":"2022-03-26T20:00:03.162966Z","iopub.status.idle":"2022-03-26T20:00:24.456108Z","shell.execute_reply.started":"2022-03-26T20:00:03.162930Z","shell.execute_reply":"2022-03-26T20:00:24.455214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model_sigmoid.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; \nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\n\nvy = history.history['val_loss']\nty = history.history['loss']\n\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:24.457566Z","iopub.execute_input":"2022-03-26T20:00:24.457819Z","iopub.status.idle":"2022-03-26T20:00:25.045827Z","shell.execute_reply.started":"2022-03-26T20:00:24.457769Z","shell.execute_reply":"2022-03-26T20:00:25.044857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saving weights \nw_after = model_sigmoid.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.figure(figsize=(12,8))\n\nplt.title(\"Weight matrices after model trained\")\n\n\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:25.047235Z","iopub.execute_input":"2022-03-26T20:00:25.048110Z","iopub.status.idle":"2022-03-26T20:00:26.767542Z","shell.execute_reply.started":"2022-03-26T20:00:25.048071Z","shell.execute_reply":"2022-03-26T20:00:26.766853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Model Building: \n* With Multi Layer perceptron (MLP),  \n* sigmoid (and softmax) activation function, \n* ADAM optimizer","metadata":{}},{"cell_type":"code","source":"model_sigmoid_adam = Sequential()\n\nmodel_sigmoid_adam.add(Dense(512, activation='sigmoid', input_shape=(input_dim,)))\nmodel_sigmoid_adam.add(Dense(128, activation='sigmoid'))\nmodel_sigmoid_adam.add(Dense(output_dim, activation='softmax'))\n\nmodel_sigmoid_adam.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:26.768776Z","iopub.execute_input":"2022-03-26T20:00:26.769190Z","iopub.status.idle":"2022-03-26T20:00:26.803211Z","shell.execute_reply.started":"2022-03-26T20:00:26.769151Z","shell.execute_reply":"2022-03-26T20:00:26.802450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_sigmoid_adam.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_sigmoid_adam.fit(x_train, Y_train, batch_size=batch_size, epochs=epoch, verbose=1, validation_data=(x_val, Y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:26.804260Z","iopub.execute_input":"2022-03-26T20:00:26.805160Z","iopub.status.idle":"2022-03-26T20:00:44.824480Z","shell.execute_reply.started":"2022-03-26T20:00:26.805124Z","shell.execute_reply":"2022-03-26T20:00:44.823775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model_sigmoid_adam.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch'); \nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:44.828772Z","iopub.execute_input":"2022-03-26T20:00:44.828996Z","iopub.status.idle":"2022-03-26T20:00:45.353697Z","shell.execute_reply.started":"2022-03-26T20:00:44.828970Z","shell.execute_reply":"2022-03-26T20:00:45.353024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Model Building: \n* With Multi Layer perceptron (MLP), \n* ReLU (and softmax) activation function, \n* SGD optimizer\n* Kerner/weight initialization using normal distribution {which uses fan_in internally}","metadata":{}},{"cell_type":"code","source":"model_relu = Sequential()\nmodel_relu.add(Dense(512, activation='relu', input_shape = (input_dim,), kernel_initializer = RandomNormal(mean=0.0, stddev=0.062, seed=None)))\nmodel_relu.add(Dense(128, activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev = 0.125, seed = None)))\nmodel_relu.add(Dense(output_dim, activation='softmax'))\n\nmodel_relu.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:45.354963Z","iopub.execute_input":"2022-03-26T20:00:45.355296Z","iopub.status.idle":"2022-03-26T20:00:45.389977Z","shell.execute_reply.started":"2022-03-26T20:00:45.355258Z","shell.execute_reply":"2022-03-26T20:00:45.389296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_relu.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:45.391382Z","iopub.execute_input":"2022-03-26T20:00:45.391670Z","iopub.status.idle":"2022-03-26T20:00:45.400453Z","shell.execute_reply.started":"2022-03-26T20:00:45.391634Z","shell.execute_reply":"2022-03-26T20:00:45.399682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_relu.fit(x_train, Y_train, \n                         batch_size=batch_size, \n                         epochs=epoch, \n                         verbose=True, \n                         validation_data = (x_val, Y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:00:45.402175Z","iopub.execute_input":"2022-03-26T20:00:45.402812Z","iopub.status.idle":"2022-03-26T20:01:06.852859Z","shell.execute_reply.started":"2022-03-26T20:00:45.402761Z","shell.execute_reply":"2022-03-26T20:01:06.852053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model_relu.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') \nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\n\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:06.854625Z","iopub.execute_input":"2022-03-26T20:01:06.854906Z","iopub.status.idle":"2022-03-26T20:01:07.503507Z","shell.execute_reply.started":"2022-03-26T20:01:06.854874Z","shell.execute_reply":"2022-03-26T20:01:07.502740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. Model Building: \n* With Multi Layer perceptron (MLP), \n* ReLU (and softmax) activation function, \n* ADAM optimizer\n* Kerner/weight initialization using normal distribution {which uses fan_in internally}","metadata":{}},{"cell_type":"code","source":"model_relu_adam = Sequential()\nmodel_relu_adam.add(Dense(512, activation='relu', input_shape = (input_dim,), kernel_initializer = RandomNormal(mean=0.0, stddev=0.062, seed=None)))\nmodel_relu_adam.add(Dense(128, activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.125, seed=None)))\nmodel_relu_adam.add(Dense(output_dim, activation='softmax'))\n\nmodel_relu_adam.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:07.505060Z","iopub.execute_input":"2022-03-26T20:01:07.505295Z","iopub.status.idle":"2022-03-26T20:01:07.538503Z","shell.execute_reply.started":"2022-03-26T20:01:07.505261Z","shell.execute_reply":"2022-03-26T20:01:07.537847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_relu_adam.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:07.539843Z","iopub.execute_input":"2022-03-26T20:01:07.540093Z","iopub.status.idle":"2022-03-26T20:01:07.548330Z","shell.execute_reply.started":"2022-03-26T20:01:07.540059Z","shell.execute_reply":"2022-03-26T20:01:07.547615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_relu_adam.fit(x_train, Y_train, \n                              batch_size=batch_size, \n                              epochs=epoch, verbose=1, \n                              validation_data=(x_val, Y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:07.549505Z","iopub.execute_input":"2022-03-26T20:01:07.550122Z","iopub.status.idle":"2022-03-26T20:01:29.042907Z","shell.execute_reply.started":"2022-03-26T20:01:07.550086Z","shell.execute_reply":"2022-03-26T20:01:29.042126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model_relu_adam.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch')\nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\n\nvy = history.history['val_loss']\nty = history.history['loss']\n\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:29.044284Z","iopub.execute_input":"2022-03-26T20:01:29.044626Z","iopub.status.idle":"2022-03-26T20:01:29.553467Z","shell.execute_reply.started":"2022-03-26T20:01:29.044588Z","shell.execute_reply":"2022-03-26T20:01:29.552804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Observations: Clear over-Fitting is being observed","metadata":{}},{"cell_type":"markdown","source":"### 6. Model Building: \n* With Multi Layer perceptron (MLP), \n* ReLU (and softmax) activation function, \n* ADAM optimizer\n* Kerner/weight initialization using normal distribution {which uses fan_in internally}\n* Batch-Normalization on Hidden layers to avoid  internal covariate shift and speed up the training process using SGD methods","metadata":{}},{"cell_type":"code","source":"model_batch = Sequential()\n\nmodel_batch.add(Dense(512, activation='relu', input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.039, seed=None)))\nmodel_batch.add(BatchNormalization()) # batch normalization\n\nmodel_batch.add(Dense(128, activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.55, seed=None)) )\nmodel_batch.add(BatchNormalization())\n\nmodel_batch.add(Dense(output_dim, activation='softmax'))\n\n\nmodel_batch.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:29.554951Z","iopub.execute_input":"2022-03-26T20:01:29.555238Z","iopub.status.idle":"2022-03-26T20:01:29.614120Z","shell.execute_reply.started":"2022-03-26T20:01:29.555199Z","shell.execute_reply":"2022-03-26T20:01:29.613452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Strongly recommend to visit the stackoverflow question on why the number of batch normalization is 2048 (512 * 4), 512 9128* 4) ? [Ans](https://stackoverflow.com/questions/42521005/how-the-number-of-parameters-associated-with-batchnormalization-layer-is-2048)\n* These 2048 parameters are in fact [gamma weights, beta weights, moving_mean(non-trainable), moving_variance(non-trainable)], each having 512 elements (the size of the input layer).","metadata":{}},{"cell_type":"code","source":"model_batch.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_batch.fit(x_train, Y_train, \n                          batch_size=batch_size, \n                          epochs=epoch, verbose=1, \n                          validation_data=(x_val, Y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:01:29.616702Z","iopub.execute_input":"2022-03-26T20:01:29.616917Z","iopub.status.idle":"2022-03-26T20:02:11.863060Z","shell.execute_reply.started":"2022-03-26T20:01:29.616893Z","shell.execute_reply":"2022-03-26T20:02:11.862259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model_batch.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') \nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:02:11.864638Z","iopub.execute_input":"2022-03-26T20:02:11.864904Z","iopub.status.idle":"2022-03-26T20:02:12.397445Z","shell.execute_reply.started":"2022-03-26T20:02:11.864867Z","shell.execute_reply":"2022-03-26T20:02:12.396806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6. Model Building: \n* With Multi Layer perceptron (MLP), \n* ReLU (and softmax) activation function, \n* ADAM optimizer\n* Kerner/weight initialization using normal distribution {which uses fan_in internally}\n* Batch-Normalization on Hidden layers to avoid  internal covariate shift and speed up the training process using SGD methods\n* Use of Dropouts to avoid overfitting","metadata":{}},{"cell_type":"code","source":"model_drop = Sequential()\n\nmodel_drop.add(Dense(512, activation='relu', input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.039, seed=None)))\nmodel_drop.add(BatchNormalization())\nmodel_drop.add(Dropout(0.5))\n\nmodel_drop.add(Dense(128, activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.55, seed=None)) )\nmodel_drop.add(BatchNormalization())\nmodel_drop.add(Dropout(0.5))\n\nmodel_drop.add(Dense(output_dim, activation='softmax'))\n\n\nmodel_drop.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:02:12.398867Z","iopub.execute_input":"2022-03-26T20:02:12.399239Z","iopub.status.idle":"2022-03-26T20:02:12.461370Z","shell.execute_reply.started":"2022-03-26T20:02:12.399196Z","shell.execute_reply":"2022-03-26T20:02:12.460526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_drop.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_drop.fit(x_train, Y_train, \n                         batch_size=batch_size, epochs=epoch, \n                         verbose=1, validation_data=(x_val, Y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:02:12.462650Z","iopub.execute_input":"2022-03-26T20:02:12.462970Z","iopub.status.idle":"2022-03-26T20:02:54.583549Z","shell.execute_reply.started":"2022-03-26T20:02:12.462935Z","shell.execute_reply":"2022-03-26T20:02:54.582665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model_drop.evaluate(x_val, Y_val, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') \nax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:02:54.587054Z","iopub.execute_input":"2022-03-26T20:02:54.587258Z","iopub.status.idle":"2022-03-26T20:02:55.118490Z","shell.execute_reply.started":"2022-03-26T20:02:54.587234Z","shell.execute_reply":"2022-03-26T20:02:55.117833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Observations: Dropout layer successfully handled over-fitting","metadata":{}},{"cell_type":"markdown","source":"### Hyper-Parameter Tuning of DL keras models using Sklearn\n* will only work on ","metadata":{}},{"cell_type":"code","source":"def best_hyperparameters(activation, optimizer):\n\n    model = Sequential()\n    model.add(Dense(512, activation=activation, input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.062, seed=None)))\n    model.add(Dense(128, activation=activation, kernel_initializer=RandomNormal(mean=0.0, stddev=0.125, seed=None)) )\n    model.add(Dense(output_dim, activation='softmax'))\n\n\n    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=optimizer)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:02:55.119785Z","iopub.execute_input":"2022-03-26T20:02:55.120130Z","iopub.status.idle":"2022-03-26T20:02:55.126652Z","shell.execute_reply.started":"2022-03-26T20:02:55.120091Z","shell.execute_reply":"2022-03-26T20:02:55.125985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_grid = {'activation': ['sigmoid', 'relu'],\n             'optimizer':['sgd', 'adam']}\n\nfrom tensorflow.keras.wrappers.scikit_learn import KerasClassifier\nfrom sklearn.model_selection import GridSearchCV\n\nmodel = KerasClassifier(build_fn=best_hyperparameters, \n                        epochs=epoch, \n                        batch_size=batch_size, verbose=0)\n\n# if you are using CPU\n# grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)\n# if you are using GPU dont use the n_jobs parameter\n\ngrid = GridSearchCV(estimator=model, param_grid=param_grid)\n\ngrid_result = grid.fit(x_train, Y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:02:55.128068Z","iopub.execute_input":"2022-03-26T20:02:55.128524Z","iopub.status.idle":"2022-03-26T20:06:53.388771Z","shell.execute_reply.started":"2022-03-26T20:02:55.128489Z","shell.execute_reply":"2022-03-26T20:06:53.388017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\n\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T20:06:53.390063Z","iopub.execute_input":"2022-03-26T20:06:53.390303Z","iopub.status.idle":"2022-03-26T20:06:53.398503Z","shell.execute_reply.started":"2022-03-26T20:06:53.390269Z","shell.execute_reply":"2022-03-26T20:06:53.397825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Observation: Using 'relu' as Activation function and 'adam' as optimizer we are getting the best results","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}