{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-04T02:19:44.484676Z","iopub.execute_input":"2022-02-04T02:19:44.48571Z","iopub.status.idle":"2022-02-04T02:19:44.505823Z","shell.execute_reply.started":"2022-02-04T02:19:44.485581Z","shell.execute_reply":"2022-02-04T02:19:44.505111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**According to Pythagoras, \" Numbers rule the universe \".**\n\nHence, in this notebook our goal is to correctly identify the numbers (digits) from a dataset of tens of thousands of handwritten images.","metadata":{}},{"cell_type":"markdown","source":"**IMPORTING LIBRARIES:**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport tensorflow as tf\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom tensorflow.keras.optimizers import RMSprop\nfrom keras.callbacks import ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:44.507268Z","iopub.execute_input":"2022-02-04T02:19:44.507468Z","iopub.status.idle":"2022-02-04T02:19:46.706593Z","shell.execute_reply.started":"2022-02-04T02:19:44.507443Z","shell.execute_reply":"2022-02-04T02:19:46.705541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**LOADING THE DATA:**","metadata":{}},{"cell_type":"code","source":"# Read training and test data files\ntrain_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:46.7079Z","iopub.execute_input":"2022-02-04T02:19:46.708329Z","iopub.status.idle":"2022-02-04T02:19:50.456599Z","shell.execute_reply.started":"2022-02-04T02:19:46.708293Z","shell.execute_reply":"2022-02-04T02:19:50.455928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:50.458827Z","iopub.execute_input":"2022-02-04T02:19:50.459343Z","iopub.status.idle":"2022-02-04T02:19:50.464465Z","shell.execute_reply.started":"2022-02-04T02:19:50.459304Z","shell.execute_reply":"2022-02-04T02:19:50.463718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_y = train_df['label'].astype('float32')\ntrain_x = train_df.drop(['label'],axis = 1).astype('int32')\ntest_x = test_df.astype('float32')\ntrain_x.shape, train_y.shape, test_x.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:50.466023Z","iopub.execute_input":"2022-02-04T02:19:50.466571Z","iopub.status.idle":"2022-02-04T02:19:50.644556Z","shell.execute_reply.started":"2022-02-04T02:19:50.466531Z","shell.execute_reply":"2022-02-04T02:19:50.643868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.countplot(train_y)\n\ntrain_y.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:50.645517Z","iopub.execute_input":"2022-02-04T02:19:50.645708Z","iopub.status.idle":"2022-02-04T02:19:50.86464Z","shell.execute_reply.started":"2022-02-04T02:19:50.645684Z","shell.execute_reply":"2022-02-04T02:19:50.864001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**NORMALIZATION & RESHAPE:**\n\n* **NORMALIZATION:**\n\nWe perform a grayscale normalization to reduce the effect of illumination's differences.Moreover the CNN converge faster on [0..1] data than on [0..255].\n\n* **RESHAPE:**\n\nTrain and test images (28px x 28px) has been stock into pandas.Dataframe as 1D vectors of 784 values. We reshape all data to 28x28x1 3D matrices.Keras requires an extra dimension in the end which correspond to channels. MNIST images are gray scaled so it use only one channel\n","metadata":{}},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\ntrain_x = train_x.values.reshape(-1,28,28,1)\n# Normalize the data\ntrain_x = train_x / 255.0\n# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\ntest_x = test_x.values.reshape(-1,28,28,1)\n# Normalize the data\ntest_x = test_x / 255.0\ntrain_x.shape, test_x.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:50.867043Z","iopub.execute_input":"2022-02-04T02:19:50.867316Z","iopub.status.idle":"2022-02-04T02:19:50.987947Z","shell.execute_reply.started":"2022-02-04T02:19:50.867282Z","shell.execute_reply":"2022-02-04T02:19:50.987135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**ONE HOT ENCODING:**\n\nOne hot encoding is one method of converting data to prepare it for an algorithm and get a better prediction. With one-hot encoding, we convert each categorical value into a new categorical column and assign a binary value of 1 or 0 to those columns. Each integer value is represented as a binary vector.\n\nHere, labels are 10 digits numbers from 0 to 9. We need to encode these lables to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0]).","metadata":{}},{"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\ntrain_y = tf.keras.utils.to_categorical(train_y,10)\ntrain_y.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:50.989569Z","iopub.execute_input":"2022-02-04T02:19:50.989872Z","iopub.status.idle":"2022-02-04T02:19:50.997269Z","shell.execute_reply.started":"2022-02-04T02:19:50.989835Z","shell.execute_reply":"2022-02-04T02:19:50.996516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Printing original labels of top 5 rows\nprint(train_df['label'].head())","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:50.998785Z","iopub.execute_input":"2022-02-04T02:19:50.999354Z","iopub.status.idle":"2022-02-04T02:19:51.00475Z","shell.execute_reply.started":"2022-02-04T02:19:50.999319Z","shell.execute_reply":"2022-02-04T02:19:51.003931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#One hot encoding of the same labels\nprint(train_y[0:5,:])","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:51.00852Z","iopub.execute_input":"2022-02-04T02:19:51.00916Z","iopub.status.idle":"2022-02-04T02:19:51.014911Z","shell.execute_reply.started":"2022-02-04T02:19:51.009122Z","shell.execute_reply":"2022-02-04T02:19:51.013909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SPLIT THE TRAIN AND THE VALIDATION SET:**","metadata":{}},{"cell_type":"code","source":"# Set the random seed\nrandom_seed = 2","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:51.016728Z","iopub.execute_input":"2022-02-04T02:19:51.01735Z","iopub.status.idle":"2022-02-04T02:19:51.022774Z","shell.execute_reply.started":"2022-02-04T02:19:51.017309Z","shell.execute_reply":"2022-02-04T02:19:51.021661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\n#from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(train_x, train_y, test_size = 0.1, random_state=random_seed)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:51.02458Z","iopub.execute_input":"2022-02-04T02:19:51.025361Z","iopub.status.idle":"2022-02-04T02:19:51.402659Z","shell.execute_reply.started":"2022-02-04T02:19:51.025306Z","shell.execute_reply":"2022-02-04T02:19:51.401915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, a small fraction (10%) became the validation set which the model is evaluated and the rest (90%) is used to train the model.","metadata":{}},{"cell_type":"markdown","source":"**VISUALIZATION OF THE DIGITS BY PLOTTING IMAGES:**\n    ","metadata":{}},{"cell_type":"code","source":"rows = 5 # defining no. of rows in figure\ncols = 6 # defining no. of colums in figure\n\nf = plt.figure(figsize=(2*cols,2*rows)) # defining a figure \n\nfor i in range(rows*cols): \n    f.add_subplot(rows,cols,i+1) # adding sub plot to figure on each iteration\n    plt.imshow(X_train[i].reshape([28,28]),cmap=\"Blues\") \n    plt.axis(\"off\")\n    #plt.title(str(Y_train[i]), y=-0.15,color=\"green\")\n   \nplt.savefig(\"digits.png\")","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:51.405108Z","iopub.execute_input":"2022-02-04T02:19:51.405488Z","iopub.status.idle":"2022-02-04T02:19:52.604955Z","shell.execute_reply.started":"2022-02-04T02:19:51.405448Z","shell.execute_reply":"2022-02-04T02:19:52.604238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**VISUALIZATION OF THE FIRST NUMBER:**","metadata":{}},{"cell_type":"code","source":"# view the first number\nplt.imshow(X_train[0].reshape(28, 28))# (rows, columns)\n# the label of the first number\nplt.title(f\"Digit: {Y_train[0]}\")\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:52.606231Z","iopub.execute_input":"2022-02-04T02:19:52.607076Z","iopub.status.idle":"2022-02-04T02:19:52.774815Z","shell.execute_reply.started":"2022-02-04T02:19:52.607032Z","shell.execute_reply":"2022-02-04T02:19:52.773934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**VISUALIZATION OF THE FIRST NUMBER IN MORE DETAIL:**","metadata":{}},{"cell_type":"code","source":"def visualize_input(img, ax):\n    ax.imshow(img, cmap='gray')\n    width, height = img.shape\n    thresh = img.max()/2.5\n    for x in range(width):\n        for y in range(height):\n            ax.annotate(str(round(img[x][y],2)), xy=(y,x),\n                        horizontalalignment='center',\n                        verticalalignment='center',\n                        color='white' if img[x][y]<thresh else 'black')\n\nfig = plt.figure(figsize = (15,15)) \nax = fig.add_subplot(111)\nvisualize_input(X_train[0].reshape(28, 28), ax)  # pass image (28, 28)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:52.777072Z","iopub.execute_input":"2022-02-04T02:19:52.777567Z","iopub.status.idle":"2022-02-04T02:19:57.294522Z","shell.execute_reply.started":"2022-02-04T02:19:52.777528Z","shell.execute_reply":"2022-02-04T02:19:57.293831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MODEL** - **CNN**\n\n* **Convolutional (Conv2D) layer**:\n\nConvolutional (Conv2D) layer is like a set of learnable filters. Each filter transforms a part of the image (defined by the kernel size) using the kernel filter. The kernel filter matrix is applied on the whole image. Filters can be seen as a transformation of the image.\n\n* **Pooling(MaxPool2D) layer:**\n\nThis layer simply acts as a downsampling filter. It looks at the 2 neighboring pixels and picks the maximal value. These are used to reduce computational cost, and to some extent also reduce overfitting.\n\n* **Dropout:**\n\nDropout is a regularization method, where a proportion of nodes in the layer are randomly ignored (setting their wieghts to zero) for each training sample. This drops randomly a propotion of the network and forces the network to learn features in a distributed way. \n\n* **relu:**\n\n'relu' is the rectifier (activation function max(0,x)). The rectifier activation function is used to add non linearity to the network.\n\n* **Flatten layer:**\n\nThe Flatten layer is use to convert the final feature maps into a one single 1D vector. This flattening step is needed so that we can make use of fully connected layers after some convolutional/maxpool layers.\n\n* **Dense layer:**\n\nDense layer is just artificial an neural networks (ANN) classifier.Finally, in the last layer(Dense(10,activation=\"softmax\")) indicates the net outputs distribution of probability of each class.\n","metadata":{}},{"cell_type":"code","source":"#Defining the model\nmodel = tf.keras.models.Sequential([\n  tf.keras.layers.Conv2D(32,(3,3),activation = 'relu', input_shape=(28,28,1)),\n  tf.keras.layers.Conv2D(32,(3,3),activation = 'relu'),\n  tf.keras.layers.MaxPooling2D(2,2),\n  tf.keras.layers.Conv2D(64,(3,3),activation = 'relu',padding = 'Same'),\n  tf.keras.layers.Conv2D(64,(3,3),activation = 'relu',padding = 'Same'),\n  tf.keras.layers.MaxPooling2D(pool_size = (2,2), strides = (2,2)),\n  tf.keras.layers.Dropout(0.25),\n  tf.keras.layers.Conv2D(64,(3,3),activation = 'relu',padding = 'Same'),\n  tf.keras.layers.Conv2D(64,(3,3),activation = 'relu',padding = 'Same'),\n  tf.keras.layers.MaxPooling2D(pool_size = (2,2), strides = (2,2)),\n  tf.keras.layers.Dropout(0.25),\n  tf.keras.layers.Flatten(),\n  tf.keras.layers.Dense(256, activation='relu'),\n  tf.keras.layers.Dense(256, activation='relu'),\n  tf.keras.layers.Dropout(0.50),\n  tf.keras.layers.Dense(10, activation='softmax')\n])\nmodel.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-04T02:19:57.295895Z","iopub.execute_input":"2022-02-04T02:19:57.296634Z","iopub.status.idle":"2022-02-04T02:19:58.282506Z","shell.execute_reply.started":"2022-02-04T02:19:57.296595Z","shell.execute_reply":"2022-02-04T02:19:58.281755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SETTING THE OPTIMIZER AND ANNEALER:**\n\n\n* **Optimizer:**\n\nThis function will iteratively improve parameters (filters kernel values, weights and bias of neurons) in order to minimise the loss.\n\n* **RMSprop():**\n\nRMSprop (with default values) is a very effective optimizer. The RMSProp update adjusts the Adagrad method in a very simple way in an attempt to reduce its aggressive, monotonically decreasing learning rate.\n\n* **Annealing method of the learning rate (LR):**\n\nThe LR is the step by which the optimizer walks through the 'loss landscape'. Its better to have a decreasing learning rate during the training to reach efficiently the global minimum of the loss function.\n\n* **ReduceLROnPlateau function:**\n\nReduce learning rate when a metric has stopped improving. Models often benefit from reducing the learning rate by a factor of 2-10 once learning stagnates. This callback monitors a quantity and if no improvement is seen for a 'patience' number of epochs, the learning rate is reduced.\n\nWith the ReduceLROnPlateau function from Keras.callbacks, let us choose to reduce the LR by half if the accuracy is not improved after 3 epochs.\n\n","metadata":{}},{"cell_type":"code","source":"# Define the optimizer\n\n#from tensorflow.keras.optimizers import RMSprop\n\noptimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-04T02:19:58.284005Z","iopub.execute_input":"2022-02-04T02:19:58.284281Z","iopub.status.idle":"2022-02-04T02:19:58.290381Z","shell.execute_reply.started":"2022-02-04T02:19:58.284245Z","shell.execute_reply":"2022-02-04T02:19:58.289617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:58.292058Z","iopub.execute_input":"2022-02-04T02:19:58.292632Z","iopub.status.idle":"2022-02-04T02:19:58.307157Z","shell.execute_reply.started":"2022-02-04T02:19:58.292596Z","shell.execute_reply":"2022-02-04T02:19:58.306358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set a learning rate annealer\n#from keras.callbacks import ReduceLROnPlateau\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:58.308151Z","iopub.execute_input":"2022-02-04T02:19:58.308774Z","iopub.status.idle":"2022-02-04T02:19:58.313409Z","shell.execute_reply.started":"2022-02-04T02:19:58.308738Z","shell.execute_reply":"2022-02-04T02:19:58.312623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 15 \nbatch_size = 86","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:58.314471Z","iopub.execute_input":"2022-02-04T02:19:58.315011Z","iopub.status.idle":"2022-02-04T02:19:58.325589Z","shell.execute_reply.started":"2022-02-04T02:19:58.314971Z","shell.execute_reply":"2022-02-04T02:19:58.324849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**DATA AUGMENTATION:**\n\nData augmentation is a technique to artificially create new training data from existing training data. This is done by applying domain-specific techniques to examples from the training data that create new and different training examples.\n\nThe idea is to alter the training data with small transformations to reproduce the variations occuring when someone is writing a digit in order to avoid overfitting problem\n\n**For example:**\n\nThe number is not centered \n\nThe scale is not the same (some who write with big/small numbers) \n\nThe image is rotated","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:19:58.326908Z","iopub.execute_input":"2022-02-04T02:19:58.32725Z","iopub.status.idle":"2022-02-04T02:19:58.426731Z","shell.execute_reply.started":"2022-02-04T02:19:58.327215Z","shell.execute_reply":"2022-02-04T02:19:58.425965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thus, by applying above transformations to our training data, we can easily double or triple the number of training examples and create a very robust model.","metadata":{}},{"cell_type":"code","source":"# Fit the model\nfinal = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-04T02:19:58.428061Z","iopub.execute_input":"2022-02-04T02:19:58.42841Z","iopub.status.idle":"2022-02-04T02:23:25.237975Z","shell.execute_reply.started":"2022-02-04T02:19:58.428372Z","shell.execute_reply":"2022-02-04T02:23:25.237133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MODEL EVALUATION:**","metadata":{}},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \n#Diffining Figure\nf = plt.figure(figsize=(20,7))\n\n#Adding Subplot 1 (For Accuracy)\nf.add_subplot(121)\n\nplt.plot(final.epoch,final.history['accuracy'],label = \"accuracy\") # Accuracy curve for training set\nplt.plot(final.epoch,final.history['val_accuracy'],label = \"val_accuracy\") # Accuracy curve for validation set\n\nplt.title(\"Accuracy Curve\",fontsize=18)\nplt.xlabel(\"Epochs\",fontsize=15)\nplt.ylabel(\"Accuracy\",fontsize=15)\nplt.grid(alpha=0.3)\nplt.legend()\n\n#Adding Subplot 1 (For Loss)\nf.add_subplot(122)\n\nplt.plot(final.epoch,final.history['loss'],label=\"loss\") # Loss curve for training set\nplt.plot(final.epoch,final.history['val_loss'],label=\"val_loss\") # Loss curve for validation set\n\nplt.title(\"Loss Curve\",fontsize=18)\nplt.xlabel(\"Epochs\",fontsize=15)\nplt.ylabel(\"Loss\",fontsize=15)\nplt.grid(alpha=0.3)\nplt.legend()\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:23:25.239698Z","iopub.execute_input":"2022-02-04T02:23:25.240059Z","iopub.status.idle":"2022-02-04T02:23:25.595312Z","shell.execute_reply.started":"2022-02-04T02:23:25.240017Z","shell.execute_reply":"2022-02-04T02:23:25.594614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(2,1)\nax[0].plot(final.history['loss'], color='g', label=\"Training loss\")\nax[0].plot(final.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(final.history['accuracy'], color='g', label=\"Training accuracy\")\nax[1].plot(final.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:23:25.596719Z","iopub.execute_input":"2022-02-04T02:23:25.597134Z","iopub.status.idle":"2022-02-04T02:23:26.016578Z","shell.execute_reply.started":"2022-02-04T02:23:25.597082Z","shell.execute_reply":"2022-02-04T02:23:26.015895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hence, we can observe that the model reaches almost 99%  accuracy on the validation dataset after 2 epochs. The validation accuracy is greater than the training accuracy almost every time during the training. That means that our model dosen't not overfit the training set.\n\nTherefore,our model is very well trained👍","metadata":{}},{"cell_type":"markdown","source":"**CONFUSION MATRIX:**\n\nA Confusion matrix is an N x N matrix used for evaluating the performance, where N is the number of target classes. The matrix compares the actual target values with those predicted by the machine learning model\n","metadata":{}},{"cell_type":"code","source":"# Look at confusion matrix \n#from sklearn.metrics import confusion_matrix\n#import intertools\n#from sklearn.model_selection import train_test_split\n#from sklearn.metrics import confusion_matrix\n#import itertools\n\n\n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Purples):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(Y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) ","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:23:26.021078Z","iopub.execute_input":"2022-02-04T02:23:26.021499Z","iopub.status.idle":"2022-02-04T02:23:27.070131Z","shell.execute_reply.started":"2022-02-04T02:23:26.021457Z","shell.execute_reply":"2022-02-04T02:23:27.069382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can observe that our CNN performs very well on all digits with few errors considering the size of the validation set.\n\nHowever, it seems that our CNN has some little troubles with the 4 digits,where they are misclassified as 9. Sometimes it is very difficult to catch the difference between 4 and 9 when curves are smooth.","metadata":{}},{"cell_type":"markdown","source":"**INVESTIGATING THE ERRORS:**","metadata":{}},{"cell_type":"code","source":"\n# Display some error results \n\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 2\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors \nmost_important_errors = sorted_dela_errors[-10:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:23:27.071576Z","iopub.execute_input":"2022-02-04T02:23:27.071951Z","iopub.status.idle":"2022-02-04T02:23:27.690715Z","shell.execute_reply.started":"2022-02-04T02:23:27.071912Z","shell.execute_reply":"2022-02-04T02:23:27.690057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**PREDICTION:**","metadata":{}},{"cell_type":"code","source":"# predict results\nresults = model.predict(test_x)\n\n# select the index with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:23:27.692134Z","iopub.execute_input":"2022-02-04T02:23:27.69264Z","iopub.status.idle":"2022-02-04T02:23:29.428891Z","shell.execute_reply.started":"2022-02-04T02:23:27.692599Z","shell.execute_reply":"2022-02-04T02:23:29.428113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SUBMISSION:**","metadata":{}},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n#submission\nsubmission.to_csv(\"submission.csv\",index=False)\nprint(\"Successfully Completed!\")","metadata":{"execution":{"iopub.status.busy":"2022-02-04T02:23:29.433523Z","iopub.execute_input":"2022-02-04T02:23:29.434069Z","iopub.status.idle":"2022-02-04T02:23:29.482164Z","shell.execute_reply.started":"2022-02-04T02:23:29.434035Z","shell.execute_reply":"2022-02-04T02:23:29.481345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**According to Shakuntala Devi, \" Numbers have life, they are not just symbols on paper \".**\n\nHence, in this notebook we have successfully identified the numbers (digits) from a dataset of tens of thousands of handwritten images.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}