{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport keras\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import RandomizedSearchCV\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-13T12:58:54.678387Z","iopub.execute_input":"2022-10-13T12:58:54.678809Z","iopub.status.idle":"2022-10-13T12:59:00.876482Z","shell.execute_reply.started":"2022-10-13T12:58:54.678772Z","shell.execute_reply":"2022-10-13T12:59:00.875274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading the data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-10-13T12:52:11.342194Z","iopub.execute_input":"2022-10-13T12:52:11.342618Z","iopub.status.idle":"2022-10-13T12:52:15.865289Z","shell.execute_reply.started":"2022-10-13T12:52:11.342586Z","shell.execute_reply":"2022-10-13T12:52:15.864034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T12:52:16.779199Z","iopub.execute_input":"2022-10-13T12:52:16.779630Z","iopub.status.idle":"2022-10-13T12:52:16.811521Z","shell.execute_reply.started":"2022-10-13T12:52:16.779594Z","shell.execute_reply":"2022-10-13T12:52:16.810257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking if the dataset is unbalanced","metadata":{}},{"cell_type":"code","source":"train['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T13:05:26.868333Z","iopub.execute_input":"2022-10-13T13:05:26.868732Z","iopub.status.idle":"2022-10-13T13:05:26.881572Z","shell.execute_reply.started":"2022-10-13T13:05:26.868702Z","shell.execute_reply":"2022-10-13T13:05:26.880139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset is unbalanced with the digit 5 having the lowest number of datapoints. So we subsample the other datapoints to have 3795 points","metadata":{}},{"cell_type":"code","source":"df0 = train[train['label'] == 0]\ndf0_balanced = df0.sample(3795)\n\ndf1 = train[train['label'] == 1]\ndf1_balanced = df1.sample(3795)\n\ndf2 = train[train['label'] == 2]\ndf2_balanced = df2.sample(3795)\n\ndf3 = train[train['label'] == 3]\ndf3_balanced = df3.sample(3795)\n\ndf4 = train[train['label'] == 4]\ndf4_balanced = df4.sample(3795)\n\ndf5 = train[train['label'] == 5]\ndf5_balanced = df5.sample(3795)\n\ndf6 = train[train['label'] == 6]\ndf6_balanced = df6.sample(3795)\n\ndf7 = train[train['label'] == 7]\ndf7_balanced = df7.sample(3795)\n\ndf8 = train[train['label'] == 8]\ndf8_balanced = df8.sample(3795)\n\ndf9 = train[train['label'] == 9]\ndf9_balanced = df9.sample(3795)\n\ntrain = pd.concat([df0, df1, df2, df3, df4, df5, df6, df7, df8, df9], axis = 0)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T13:13:20.551931Z","iopub.execute_input":"2022-10-13T13:13:20.552847Z","iopub.status.idle":"2022-10-13T13:13:21.044395Z","shell.execute_reply.started":"2022-10-13T13:13:20.552809Z","shell.execute_reply":"2022-10-13T13:13:21.043181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shuffling the data\ntrain = train.sample(frac = 1)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T13:13:29.038600Z","iopub.execute_input":"2022-10-13T13:13:29.039051Z","iopub.status.idle":"2022-10-13T13:13:29.281247Z","shell.execute_reply.started":"2022-10-13T13:13:29.039015Z","shell.execute_reply":"2022-10-13T13:13:29.280077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train['label']\ny_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T13:13:53.100946Z","iopub.execute_input":"2022-10-13T13:13:53.101386Z","iopub.status.idle":"2022-10-13T13:13:53.110631Z","shell.execute_reply.started":"2022-10-13T13:13:53.101337Z","shell.execute_reply":"2022-10-13T13:13:53.109511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train.drop('label', axis = 1)\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T13:14:17.516436Z","iopub.execute_input":"2022-10-13T13:14:17.516816Z","iopub.status.idle":"2022-10-13T13:14:17.644080Z","shell.execute_reply.started":"2022-10-13T13:14:17.516786Z","shell.execute_reply":"2022-10-13T13:14:17.642823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Scaling the data**","metadata":{}},{"cell_type":"code","source":"# Scaling the data\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T13:14:38.688630Z","iopub.execute_input":"2022-10-13T13:14:38.689780Z","iopub.status.idle":"2022-10-13T13:14:39.278664Z","shell.execute_reply.started":"2022-10-13T13:14:38.689712Z","shell.execute_reply":"2022-10-13T13:14:39.277391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:16:28.085204Z","iopub.execute_input":"2022-10-13T17:16:28.085638Z","iopub.status.idle":"2022-10-13T17:16:28.096081Z","shell.execute_reply.started":"2022-10-13T17:16:28.085601Z","shell.execute_reply":"2022-10-13T17:16:28.094684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.to_numpy().shape","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:19:03.946910Z","iopub.execute_input":"2022-10-13T17:19:03.947331Z","iopub.status.idle":"2022-10-13T17:19:03.953898Z","shell.execute_reply.started":"2022-10-13T17:19:03.947295Z","shell.execute_reply":"2022-10-13T17:19:03.952959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = y_train\ny_train = y_train.to_numpy().reshape(42000,1)\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:20:01.057676Z","iopub.execute_input":"2022-10-13T17:20:01.058102Z","iopub.status.idle":"2022-10-13T17:20:01.065227Z","shell.execute_reply.started":"2022-10-13T17:20:01.058066Z","shell.execute_reply":"2022-10-13T17:20:01.064415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(n_hidden = 1, n_neurons=784, learning_rate = 0.01, input_shape = [784], activation_func = 'relu', loss_func = 'sparse_categorical_crossentropy', optimizer = 'Adam'):\n  model = keras.models.Sequential()\n  model.add(keras.layers.InputLayer(input_shape = input_shape))\n  \n  for layer in range(n_hidden):\n\n    # We add a batch normalization layer for all layers with activations function other than\n    # selu as selu is self normalizing\n    if activation_func != 'selu':\n      \n      # Setting the initializer strategy as he_normal in case of relu and its variants\n      if (activation_func in [ tf.keras.layers.PReLU(), tf.keras.layers.LeakyReLU(), 'relu', 'elu']):\n        init = 'he_normal'\n      \n      # Glorot uniforn in case of tanh and sigmoid\n      else:\n        init = tf.keras.initializers.GlorotUniform()\n      \n      # Adding a batch normalization layer\n      model.add(keras.layers.BatchNormalization())\n    \n    else:\n      # LeCun initializer works well with selu activation function\n      init = tf.keras.initializers.LecunUniform()\n\n    model.add(keras.layers.Dense(n_neurons, activation = activation_func, kernel_initializer= init))  \n  # Adding the output neuron\n  model.add(keras.layers.Dense(10, activation = 'sigmoid'))\n  \n  # Compiling the model\n  optimizer = optimizer\n  model.compile(loss = loss_func, optimizer = optimizer)\n  return model","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:20:23.295624Z","iopub.execute_input":"2022-10-13T17:20:23.296009Z","iopub.status.idle":"2022-10-13T17:20:23.307687Z","shell.execute_reply.started":"2022-10-13T17:20:23.295979Z","shell.execute_reply":"2022-10-13T17:20:23.306470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hyperparameter tuning using Randomized search\n\nThe hyper-parameters were tuned using sklearn’s RandomizedSearchCV. The hunt for the best values was done by inputting the model and a dictionary of possible values in the RandomizedSearchcv object such that while searching for one hyper-parameter the others were kept fixed enabling us to get a good understanding of how the model’s performance changes based on the parameter alone. Among the list of values for the hyper-parameter being tuned the n_iter value was set to 4 which means out of the given list of possible values, 4 will be sampled at random. Based on these 4 values the models were tested using 3-fold cross-validation with negative mean absolute error as the scoring function. The average training and validation score is then plotted to give an indication of which hyperparameter is best suited.","metadata":{}},{"cell_type":"code","source":"# Function for plotting the results of the random search\ndef plot_random_search(cv_results, parameter_name, display_name):\n    # Get the mean train and test scores\n    y_1 = cv_results['mean_test_score']\n    y_2 = cv_results['mean_train_score']\n\n    # Plot random search scores\n    x = []\n    for i in range(len(results.cv_results_['params'])):\n        x.append(str(results.cv_results_['params'][i][parameter_name]))\n    plt.title('Mean accuracy per parameter')\n    plt.ylabel('Mean accuracy')\n    plt.plot(x, y_1, linestyle='--', marker='o', label='Validation')\n    plt.plot(x, y_2, linestyle='-', marker='^',label='Train' )\n    plt.xlabel(display_name)\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:20:27.024382Z","iopub.execute_input":"2022-10-13T17:20:27.025074Z","iopub.status.idle":"2022-10-13T17:20:27.032303Z","shell.execute_reply.started":"2022-10-13T17:20:27.025035Z","shell.execute_reply":"2022-10-13T17:20:27.031433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using the sklearn wrapper for cross validation\nreg_model = keras.wrappers.scikit_learn.KerasClassifier(build_model, batch_size = 32, epochs = 5)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:20:36.655994Z","iopub.execute_input":"2022-10-13T17:20:36.656898Z","iopub.status.idle":"2022-10-13T17:20:36.662911Z","shell.execute_reply.started":"2022-10-13T17:20:36.656849Z","shell.execute_reply":"2022-10-13T17:20:36.661784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Activation Functions","metadata":{}},{"cell_type":"code","source":"param_distribs = {\n    'activation_func': [ tf.keras.layers.PReLU(), tf.keras.layers.LeakyReLU(),'sigmoid', 'tanh', 'relu', 'elu', 'selu']\n}\n\nrandom_search = RandomizedSearchCV(reg_model, \n                                   param_distribs, \n                                   cv = 3, \n                                   scoring = 'accuracy',\n                                   error_score = 'raise', \n                                   n_iter = 4,\n                                   n_jobs = -1,\n                                   return_train_score = True,\n                                   random_state = 42)\n\nresults = random_search.fit(X_train, y_train, verbose = 1)\n\n# Printing the best parameter\nprint('The best value is: ', random_search.best_params_)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:21:11.567054Z","iopub.execute_input":"2022-10-13T17:21:11.567965Z","iopub.status.idle":"2022-10-13T17:24:47.497597Z","shell.execute_reply.started":"2022-10-13T17:21:11.567930Z","shell.execute_reply":"2022-10-13T17:24:47.496440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_random_search(results.cv_results_, 'activation_func', 'Activation functions')","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:24:47.500085Z","iopub.execute_input":"2022-10-13T17:24:47.500910Z","iopub.status.idle":"2022-10-13T17:24:47.717045Z","shell.execute_reply.started":"2022-10-13T17:24:47.500865Z","shell.execute_reply":"2022-10-13T17:24:47.715944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_distribs = {\n    'activation_func' : [tf.keras.layers.PReLU()],\n    'n_hidden': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n}\n\nrandom_search = RandomizedSearchCV(reg_model, \n                                   param_distribs, \n                                   cv = 3, \n                                   scoring = 'accuracy',\n                                   error_score = 'raise', \n                                   n_iter = 4,\n                                   n_jobs = -1,\n                                   return_train_score = True,\n                                   random_state = 42)\n\nresults = random_search.fit(X_train, y_train, verbose = 1)\n\n# Printing the best parameter\nprint('The best value is: ', random_search.best_params_)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:25:51.302872Z","iopub.execute_input":"2022-10-13T17:25:51.303303Z","iopub.status.idle":"2022-10-13T17:37:23.302200Z","shell.execute_reply.started":"2022-10-13T17:25:51.303259Z","shell.execute_reply":"2022-10-13T17:37:23.300602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_random_search(results.cv_results_, 'n_hidden', 'Number of hidden layers')","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:37:24.092409Z","iopub.execute_input":"2022-10-13T17:37:24.093141Z","iopub.status.idle":"2022-10-13T17:37:24.282510Z","shell.execute_reply.started":"2022-10-13T17:37:24.093103Z","shell.execute_reply":"2022-10-13T17:37:24.281480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Number of Neurons per layer**","metadata":{}},{"cell_type":"code","source":"param_distribs = {\n    'activation_func' : [tf.keras.layers.LeakyReLU()],\n    'n_hidden': [1],\n    'n_neurons': [400, 500, 785, 900, 1000]\n\n}\n\nrandom_search = RandomizedSearchCV(reg_model, \n                                   param_distribs, \n                                   cv = 3, \n                                   scoring = 'accuracy',\n                                   error_score = 'raise', \n                                   n_iter = 4,\n                                   n_jobs = -1,\n                                   return_train_score = True,\n                                   random_state = 42)\n\nresults = random_search.fit(X_train, y_train, verbose = 1)\n\n# Printing the best parameter\nprint('The best value is: ', random_search.best_params_)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:38:56.592124Z","iopub.execute_input":"2022-10-13T17:38:56.592947Z","iopub.status.idle":"2022-10-13T17:42:10.424204Z","shell.execute_reply.started":"2022-10-13T17:38:56.592904Z","shell.execute_reply":"2022-10-13T17:42:10.422932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_random_search(results.cv_results_, 'n_neurons', 'Number of neurons per layer')","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:45:38.947743Z","iopub.execute_input":"2022-10-13T17:45:38.948211Z","iopub.status.idle":"2022-10-13T17:45:39.156269Z","shell.execute_reply.started":"2022-10-13T17:45:38.948166Z","shell.execute_reply":"2022-10-13T17:45:39.155350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Similar to the number of layers, the number of nodes in a layer are also important. Too many could overfit the data by detecting unnecessary patterns among the variables itself while too few tend to result in a high bias due to underfitting. In this case we choose 125 as it has the highest training and validation score compared to the others.","metadata":{}},{"cell_type":"markdown","source":"**Optimizer**","metadata":{}},{"cell_type":"code","source":"param_distribs = {\n    'n_hidden' : [1],\n    'n_neurons': [400],\n    'activation_func': [tf.keras.layers.LeakyReLU()],\n    'optimizer': ['Adam','RMSprop', 'Adagrad', 'Adadelta', 'Adamax', 'Nadam']\n}\n\nrandom_search = RandomizedSearchCV(reg_model, \n                                   param_distribs, \n                                   cv = 3, \n                                   scoring = 'accuracy',\n                                   error_score = 'raise', \n                                   n_iter = 4,\n                                   n_jobs = -1,\n                                   return_train_score = True,\n                                   random_state = 42)\n\nresults = random_search.fit(X_train, y_train, verbose = 1)\n\n# Printing the best parameter\nprint('The best value is: ', random_search.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:49:12.601313Z","iopub.execute_input":"2022-10-13T17:49:12.602206Z","iopub.status.idle":"2022-10-13T17:52:22.022255Z","shell.execute_reply.started":"2022-10-13T17:49:12.602153Z","shell.execute_reply":"2022-10-13T17:52:22.021122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_random_search(results.cv_results_, 'optimizer', 'Optimizer')","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:53:03.837255Z","iopub.execute_input":"2022-10-13T17:53:03.838558Z","iopub.status.idle":"2022-10-13T17:53:04.039488Z","shell.execute_reply.started":"2022-10-13T17:53:03.838490Z","shell.execute_reply":"2022-10-13T17:53:04.038183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Batch size**","metadata":{}},{"cell_type":"code","source":"param_distribs = {\n    'n_hidden' : [1],\n    'n_neurons': [400],\n    'activation_func': [tf.keras.layers.LeakyReLU()],\n    'optimizer': ['RMSprop'],\n    'loss_func' : ['sparse_categorical_crossentropy'],\n    'batch_size': [5, 10, 20, 32, 40, 50, 60]\n\n\n}\n\nrandom_search = RandomizedSearchCV(reg_model, \n                                   param_distribs, \n                                   cv = 3, \n                                   scoring = 'accuracy',\n                                   error_score = 'raise', \n                                   n_iter = 4,\n                                   n_jobs = -1,\n                                   return_train_score = True,\n                                   random_state = 42)\n\nresults = random_search.fit(X_train, y_train, verbose = 1)\n\n# Printing the best parameter\nprint('The best value is: ', random_search.best_params_)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-13T17:57:19.868506Z","iopub.execute_input":"2022-10-13T17:57:19.868935Z","iopub.status.idle":"2022-10-13T18:04:32.428797Z","shell.execute_reply.started":"2022-10-13T17:57:19.868899Z","shell.execute_reply":"2022-10-13T18:04:32.427729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_random_search(results.cv_results_, 'batch_size', 'Batch size')","metadata":{"execution":{"iopub.status.busy":"2022-10-13T18:04:32.433003Z","iopub.execute_input":"2022-10-13T18:04:32.433389Z","iopub.status.idle":"2022-10-13T18:04:32.635678Z","shell.execute_reply.started":"2022-10-13T18:04:32.433338Z","shell.execute_reply":"2022-10-13T18:04:32.634402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Initial Learning rate**\n\nIn this section we will be using Random search to see which learning rate helps train the model fastest in the first few epochs of training. We will not be using a constant learning rate as this might cause the model to train very slowly if its small or cause it oscillate and never converge to the global minimum.\n\nTo avoid this we will be using a learning rate scheduler while training the final model. This helps take advantage of the high initial learning rate at the beginning and progressively drop the its value as progresses resulting in a fine tuned search for the minumum.","metadata":{}},{"cell_type":"code","source":"param_distribs = {\n    'n_hidden' : [1],\n    'n_neurons': [400],\n    'activation_func': [tf.keras.layers.LeakyReLU()],\n    'optimizer': ['RMSprop'],\n    'loss_func' : ['sparse_categorical_crossentropy'],\n    'batch_size': [50],\n    'learning_rate' : [0.1, 0.01, 0.001, 0.0001]\n\n\n}\n\nrandom_search = RandomizedSearchCV(reg_model, \n                                   param_distribs, \n                                   cv = 3, \n                                   scoring = 'accuracy',\n                                   error_score = 'raise', \n                                   n_iter = 4,\n                                   n_jobs = -1,\n                                   return_train_score = True,\n                                   random_state = 42)\n\nresults = random_search.fit(X_train, y_train, verbose = 1)\n\n# Printing the best parameter\nprint('The best value is: ', random_search.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T18:06:46.421870Z","iopub.execute_input":"2022-10-13T18:06:46.422301Z","iopub.status.idle":"2022-10-13T18:09:16.224163Z","shell.execute_reply.started":"2022-10-13T18:06:46.422266Z","shell.execute_reply":"2022-10-13T18:09:16.222836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_random_search(results.cv_results_, 'learning_rate', 'Learning Rate')","metadata":{"execution":{"iopub.status.busy":"2022-10-13T18:09:16.226520Z","iopub.execute_input":"2022-10-13T18:09:16.227086Z","iopub.status.idle":"2022-10-13T18:09:16.431097Z","shell.execute_reply.started":"2022-10-13T18:09:16.227047Z","shell.execute_reply":"2022-10-13T18:09:16.429917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Epochs**\n\nFor tuning the epochs we will be using the Early stopping callback while training the final model based on the above tuned hyperparameters. We will choose a sufficiently high number of epochs and start trainning the model with the early stopping callback with patience = 10 indicating that if the validation loss doesnt improve for 10 successive epochs the training will stop. This will help us save time and resources that would be lost in using grid search or Randomised search for training a large number of epochs.","metadata":{}},{"cell_type":"markdown","source":"**Final model based on hyperparameter tuning**\n\nHere specify the weight initializer, as the Glorot initializer as it works well with the sigmoid activation function.","metadata":{}},{"cell_type":"code","source":"def final_model(n_hidden = 1, n_neurons=400, learning_rate = 0.01, input_shape = [784], activation_func = 'relu', loss_func = 'categorical_crossentropy', optimizer = tf.keras.optimizers.RMSprop()):\n  model = keras.models.Sequential()\n  model.add(keras.layers.InputLayer(input_shape = input_shape))\n  \n  for layer in range(n_hidden):\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Dense(n_neurons, \n                                 activation = activation_func,\n                                 kernel_initializer = 'he_normal')                                 )  \n  # Adding the output neuron\n  model.add(keras.layers.Dense(10, activation = 'softmax'))\n  \n  # Compiling the model\n  optimizer = optimizer\n  optimizer.learning_rate = learning_rate\n  model.compile(loss = loss_func, optimizer = optimizer, metrics = ['accuracy'])\n  \n  return model","metadata":{"execution":{"iopub.status.busy":"2022-10-13T18:46:25.719061Z","iopub.execute_input":"2022-10-13T18:46:25.719951Z","iopub.status.idle":"2022-10-13T18:46:25.729860Z","shell.execute_reply.started":"2022-10-13T18:46:25.719909Z","shell.execute_reply":"2022-10-13T18:46:25.728571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Developing the final model based on the hyper parameter tunining**\n\nKeras helps you specify a list of objects to be called at the start and end of training with the help of callbacks.\n\nWe will get the best results out of the epochs parameter by using the Early stopping callback which stops training when the performace of the model does not improve for certain number of iteration. In our case we will fix this value at 10.\n\nAs mentioned before we will also be using a learning rate scheduler with the help of callbacks. The one we will be using is such that whenever the perfomance of a model plateaus (does not improve for 5 successive iterations) it reduces the learning rate by a predetermined factor (0.5).","metadata":{}},{"cell_type":"code","source":"# Dividing the training data again to obtain the vaidation data for using thcall backs\nX_train_final, X_valid, y_train_final, y_valid = train_test_split(X_train,\n                                                                  y_train,\n                                                                  test_size = 0.2,\n                                                                  shuffle = True,\n                                                                  random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T18:26:36.524764Z","iopub.execute_input":"2022-10-13T18:26:36.525315Z","iopub.status.idle":"2022-10-13T18:26:38.006439Z","shell.execute_reply.started":"2022-10-13T18:26:36.525273Z","shell.execute_reply":"2022-10-13T18:26:38.005149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_final.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-13T18:27:16.619448Z","iopub.execute_input":"2022-10-13T18:27:16.619906Z","iopub.status.idle":"2022-10-13T18:27:16.627972Z","shell.execute_reply.started":"2022-10-13T18:27:16.619867Z","shell.execute_reply":"2022-10-13T18:27:16.626420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining and compiling the model\nneural_network = final_model(n_hidden = 2,\n                            n_neurons = 400,\n                            activation_func = tf.keras.layers.LeakyReLU(),\n                            optimizer = tf.keras.optimizers.RMSprop(),\n                            loss_func = 'sparse_categorical_crossentropy',\n                            learning_rate = 0.01)\n\n# Callbacks for better epoch tuning and implementing learning rate scheduling\nearly_stop = tf.keras.callbacks.EarlyStopping(patience = 10,\n                                              mode = 'max',\n                                              verbose = 1,\n                                              restore_best_weights = True,\n                                              monitor = 'val_accuracy')\n\n# lr scheduling\nlr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(factor = 0.5, \n                                                    patience = 5,\n                                                   monitor = 'val_accuracy',\n                                                   mode = 'max')\n\n\n\n# Fitting the model\nhistory = neural_network.fit(X_train_final, \n                             y_train_final, \n                             epochs = 150, \n                             batch_size = 50,\n                             validation_data = (X_valid, y_valid),\n                             callbacks = [early_stop,lr_scheduler],\n                             verbose = 1\n                             )","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:38:10.601737Z","iopub.execute_input":"2022-10-13T19:38:10.602311Z","iopub.status.idle":"2022-10-13T19:42:14.052639Z","shell.execute_reply.started":"2022-10-13T19:38:10.602262Z","shell.execute_reply":"2022-10-13T19:42:14.051497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training and validation loss curves**","metadata":{}},{"cell_type":"code","source":"# Plotting the training and testing losses\nplt.plot(history.history['val_accuracy'], linestyle='-', label='Validation')\nplt.plot(history.history['accuracy'], linestyle='-',label='Train' )\nplt.xlabel('epochs')\nplt.ylabel('Accuracy')\nplt.title('Training and validation Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:47:41.369729Z","iopub.execute_input":"2022-10-13T19:47:41.370240Z","iopub.status.idle":"2022-10-13T19:48:34.127656Z","shell.execute_reply.started":"2022-10-13T19:47:41.370200Z","shell.execute_reply":"2022-10-13T19:48:34.126385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model evaluation**","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\nX_test = scaler.fit_transform(test)\nX_test","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:34.129901Z","iopub.execute_input":"2022-10-13T19:48:34.130395Z","iopub.status.idle":"2022-10-13T19:48:34.599844Z","shell.execute_reply.started":"2022-10-13T19:48:34.130332Z","shell.execute_reply":"2022-10-13T19:48:34.598918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.argmax(neural_network.predict(X_test), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:37.374918Z","iopub.execute_input":"2022-10-13T19:48:37.375351Z","iopub.status.idle":"2022-10-13T19:48:38.702055Z","shell.execute_reply.started":"2022-10-13T19:48:37.375304Z","shell.execute_reply":"2022-10-13T19:48:38.701043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:38.703272Z","iopub.execute_input":"2022-10-13T19:48:38.706419Z","iopub.status.idle":"2022-10-13T19:48:38.715410Z","shell.execute_reply.started":"2022-10-13T19:48:38.706362Z","shell.execute_reply":"2022-10-13T19:48:38.714367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.index","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:38.717255Z","iopub.execute_input":"2022-10-13T19:48:38.717818Z","iopub.status.idle":"2022-10-13T19:48:38.727563Z","shell.execute_reply.started":"2022-10-13T19:48:38.717773Z","shell.execute_reply":"2022-10-13T19:48:38.726353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dictionary = {\"ImageId\":test.index,\n             \"label\": predictions}\nfinal = pd.DataFrame(dictionary)\nfinal.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:38.728920Z","iopub.execute_input":"2022-10-13T19:48:38.729673Z","iopub.status.idle":"2022-10-13T19:48:38.743848Z","shell.execute_reply.started":"2022-10-13T19:48:38.729627Z","shell.execute_reply":"2022-10-13T19:48:38.742724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:38.745186Z","iopub.execute_input":"2022-10-13T19:48:38.745551Z","iopub.status.idle":"2022-10-13T19:48:38.798981Z","shell.execute_reply.started":"2022-10-13T19:48:38.745519Z","shell.execute_reply":"2022-10-13T19:48:38.797739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.index","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:38.801893Z","iopub.execute_input":"2022-10-13T19:48:38.802457Z","iopub.status.idle":"2022-10-13T19:48:38.809035Z","shell.execute_reply.started":"2022-10-13T19:48:38.802420Z","shell.execute_reply":"2022-10-13T19:48:38.807782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([np.array(test.index)+1, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='ImageId,label',\n    comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-10-13T19:48:38.810925Z","iopub.execute_input":"2022-10-13T19:48:38.811415Z","iopub.status.idle":"2022-10-13T19:48:40.091853Z","shell.execute_reply.started":"2022-10-13T19:48:38.811348Z","shell.execute_reply":"2022-10-13T19:48:40.090582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}