{"nbformat": 4, "cells": [{"cell_type": "markdown", "metadata": {"_uuid": "c3ea42552616e1b0c47d5b0f3009fa2c46f05137", "_cell_guid": "303e2ef3-6144-43a5-92e6-473ed334bde0"}, "source": ["# Digit Recognizer\n", "\n", "## A BEGINNER'S GUIDE\n", "\n", "Using\n", "- Multi-layer Perceptron (MLP) Model\n", "- Convolutional Neural Network (CNN) Model\n", "- Keras Neural Network Library"]}, {"cell_type": "markdown", "metadata": {"_uuid": "3c6d88291c8f35b1739288bf67569467585e2251", "_cell_guid": "e44a8854-72f1-4e08-9e89-136397d20121"}, "source": ["## Import Libraries"]}, {"cell_type": "code", "metadata": {"_uuid": "ce8285ed5a16a3ff8a79f44dad88edb062e27757", "_cell_guid": "8f08cbfc-2762-481b-a153-113f88b5bb81"}, "execution_count": null, "source": ["import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "import seaborn as sns\n", "sns.set() # setting seaborn default for plots\n", "\n", "from sklearn.metrics import confusion_matrix\n", "from sklearn.model_selection import train_test_split\n", "\n", "from keras.utils import np_utils\n", "from keras.datasets import mnist\n", "\n", "# for Multi-layer Perceptron (MLP) model\n", "from keras.models import Sequential\n", "from keras.layers import Dense\n", "\n", "# for Convolutional Neural Network (CNN) model\n", "from keras.layers import Dropout, Flatten\n", "from keras.layers.convolutional import Conv2D\n", "from keras.layers.convolutional import MaxPooling2D\n", "\n", "# fix for issue: https://github.com/fchollet/keras/issues/2681\n", "from keras import backend as K\n", "K.set_image_dim_ordering('th')"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "c2007a628239bf43f72e1612ac7ee92ca58c6ee9", "_cell_guid": "e3abd515-7ef0-45c8-9d04-d375c584bfe7"}, "source": ["## Loading Train and Test datasets"]}, {"cell_type": "code", "metadata": {"_uuid": "d27d400b26f06748e0c0820b56ed57d521e9749a", "_cell_guid": "798dbf7f-6c6f-459f-a0ee-874f49cbafb7"}, "execution_count": null, "source": ["train = pd.read_csv('../input/train.csv')\n", "print (train.shape)\n", "train.head()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "cfc8c4c1b932623be4861b97f881a0dd574965a6", "_cell_guid": "0d9624a2-0f19-4b9f-b220-4675ba7c12c6"}, "execution_count": null, "source": ["test = pd.read_csv('../input/test.csv')\n", "print (test.shape)\n", "test.head()"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_uuid": "00a684c1f99e083b4f30aecc63e51c46ea76060d", "_cell_guid": "0c1d4d59-e73b-4bf5-b4cc-42270f8b339d"}, "execution_count": null, "source": ["y_train = train['label']\n", "X_train = train.drop(labels=['label'], axis=1)\n", "X_test = test\n", "\n", "print (y_train.value_counts())"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "00822ae869b79688fe1ab4605b2dffbad4c5f36d", "_cell_guid": "7efd1947-ac52-42af-8789-3f71c33c704e"}, "execution_count": null, "source": ["sns.countplot(y_train)"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_uuid": "fbbe027e428b0b7ad76a543a1a61e2bf60bb14c0", "_cell_guid": "8980b890-4222-4a00-a561-b29b61ce1a8a"}, "execution_count": null, "source": ["X_train.head()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "0f48c660e6a83c73dbef6358683cc8bfd915417e", "_cell_guid": "023400e4-0453-475f-8f11-2be1ffef326b"}, "execution_count": null, "source": ["# check for corrupted images in the datasets\n", "# i.e. check if there are any empty pixel values\n", "print (X_train.isnull().any().sum())\n", "print (X_test.isnull().any().sum())"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "f7ab1ba3565b248ba5847cfc74bccb2d08b577a8", "_cell_guid": "dd8faf1b-86d1-4c68-b8ba-527489f6f23a"}, "source": ["## Get values of data"]}, {"cell_type": "code", "metadata": {"_uuid": "e2e2cd35ee897ded21fee2b0a4879f1f0ad12286", "collapsed": true, "_cell_guid": "6407af49-6891-4e6d-a111-f487f62dd9e8"}, "execution_count": null, "source": ["X_train = X_train.values.astype('float32') # pixel values of all images in train set\n", "y_train = y_train.values.astype('int32') # labels of all images\n", "X_test = test.values.astype('float32') # pixel values of all images in test set"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "b6f028a562e0696bd79083fc12e7ce782ed21f34", "_cell_guid": "73a6aa13-dab2-4a07-9423-bd64a4266300"}, "source": ["## Viewing shape and content of data"]}, {"cell_type": "code", "metadata": {"_uuid": "2d57d946a312686fa2f66b0fb63a9914a9a1b616", "_cell_guid": "73378005-5905-4a67-bae2-f9f0f5800c8f"}, "execution_count": null, "source": ["print (X_train.shape)\n", "print (y_train.shape)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "5a897655c1f905b17ec223252ca49af056068b7f", "_cell_guid": "dfe3b427-6af3-4215-bec3-354db4919cca"}, "execution_count": null, "source": ["print (y_train[0])\n", "print (X_train[0])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "e934569c7c4a2525be3daf66b640ac32a48116fb", "_cell_guid": "59083f89-072e-4846-a563-b8868b9e618d"}, "source": ["## Plotting images and their class values"]}, {"cell_type": "code", "metadata": {"_uuid": "9defb2e0a49cdacda41f666098d142241227de63", "_cell_guid": "f4b8283c-bba6-4595-ba21-df833bc72028"}, "execution_count": null, "source": ["plt.figure(figsize=[20,8])\n", "for i in range(6):\n", "    plt.subplot(1,6,i+1)\n", "    # Here, we reshape the 784 pixels vector values into 28x28 pixels image\n", "    plt.imshow(X_train[i].reshape(28, 28), cmap='gray', interpolation='none')\n", "    plt.title(\"Class {}\".format(y_train[i]))"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "33e6959a06d2da64e7064a0d498228c847ef69eb", "collapsed": true, "_cell_guid": "d4320d8d-07a1-4273-8f42-302e8cf4a911"}, "execution_count": null, "source": ["# fix random seed for reproducibility\n", "random_seed = 7\n", "np.random.seed(random_seed)"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "f07a61a913595b1a1fefc52c85e1ba8a3bdac96b", "_cell_guid": "bca88404-1a13-4c59-9e2f-3cdeb6365f19"}, "source": ["## Normalizing input values\n", "\n", "As we can see above, the pixel values for each image are gray scaled between 0 and 255. We now, normalize those values from 0-255 to 0-1."]}, {"cell_type": "code", "metadata": {"_uuid": "bc8f112c884091bfb6115f14c54b190f463381ae", "_cell_guid": "3910c46c-541b-405e-8c6e-10eff584147b"}, "execution_count": null, "source": ["# pixel values are gray scale between 0 and 255\n", "# normalize inputs from 0-255 to 0-1\n", "X_train = X_train / 255\n", "X_test = X_test / 255\n", "print (X_train[1])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "835a4ce3ea924d82adad295a9db3322e3b466e6e", "_cell_guid": "bf577d50-23a0-4621-a176-52d2d0c44103"}, "source": ["## Converting target variable values into one-hot format\n", "\n", "The output/target variable is in the format 0 to 9. As this is a multi-class classification problem, we convert the output class values into one-hot format which is simply a binary matrix, i.e.\n", "\n", ">value 0 will be converted to one-hot format as [1, 0, 0, 0, 0, 0, 0, 0, 0]\n", ">\n", ">value 1 will be converted to one-hot format as [0, 1, 0, 0, 0, 0, 0, 0, 0]\n", ">\n", ">value 2 will be converted to one-hot format as [0, 0, 1, 0, 0, 0, 0, 0, 0]\n", ">\n", ">and so on...\n"]}, {"cell_type": "code", "metadata": {"_uuid": "55b53264c9ce1f4419062f6a0be81b8a97b9c37b", "_cell_guid": "e57de5d2-92e5-4c8a-a928-a95e3f2cd777"}, "execution_count": null, "source": ["print (y_train.shape)\n", "print (y_train[0])"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "015faba37f6c1556b46f4e84e59389246eeb2d85", "collapsed": true, "_cell_guid": "0d0f72bc-7f94-462c-abb5-db0b2eb27257"}, "execution_count": null, "source": ["# one hot encode outputs\n", "# note that we have new variables with capital Y\n", "# Y_train is different than y_train\n", "Y_train = np_utils.to_categorical(y_train)\n", "num_classes = Y_train.shape[1]"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "1160de0a5e10ef4ed12b73330d6297b7dea75d79", "_cell_guid": "76652c57-4ec7-47b0-b839-87d239651388"}, "execution_count": null, "source": ["print (y_train.shape, Y_train.shape)\n", "print (y_train[0], Y_train[0])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "435915ad62f207d2a1ef2b0140ea98c8401a7707", "_cell_guid": "4b86afa2-a647-4026-883b-42aa60eb9af9"}, "source": ["## Splitting train dataset into training and validation set\n", "\n", "We split the train dataset into two parts in 9:1 ratio. 90% will be the actual training set and the remaining 10% will be the validation/testing set. We train our model using the training set and test the accuracy of the model using the validation set."]}, {"cell_type": "code", "metadata": {"_uuid": "1c27b184d3f5cad1346d5472777f38bdd09b8405", "collapsed": true, "_cell_guid": "96c55494-c8df-49dd-a937-0679db144ad9"}, "execution_count": null, "source": ["# Split the entire training set into two separate sets: Training set and Validation set\n", "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.10, random_state=random_seed)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "0d1fade9d1c93c07c27af2703be1610682eb72e3", "_cell_guid": "52db5f01-f4f4-475f-96e0-99e9832fc9eb"}, "execution_count": null, "source": ["print (X_train.shape, Y_train.shape, X_val.shape, Y_val.shape)\n", "num_pixels = X_train.shape[1]"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "f7ac9d104d368014bc36b682a3c14c304d73f561", "_cell_guid": "49bf7399-454f-446b-9d0c-434dcd8cc072"}, "execution_count": null, "source": ["print (Y_val)\n", "# converting one-hot format of digits to normal values/labels\n", "y_val = np.argmax(Y_val, 1) # reverse of to_categorical\n", "print (y_val)\n", "# Note that: capital Y_val contains values in one-hot format and small y_val contains normal digit values"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "ca6d92227f9568d528c3f783cb09dca3cf32582a", "_cell_guid": "de74c7e9-5a40-43c4-b0af-681ee9fd220e"}, "source": ["## Define Simple Perceptron Model\n", "\n", "Generally, neural networks have the following properties:\n", "- an input layer as a single vector\n", "- zero or multiple hidden layers after input layer\n", "- an output layer after hidden layers which represents class scores in classification problem\n", "- each neuron in a hidden layer is fully connected to all neurons in the previous layer\n", "- neurons in a single layer function independently and do not have any connection with other neurons of the same layer\n", "\n", "A **single-layer perceptron model** is the simplest kind of neural network where there are only two layers: ***input layer*** and ***output layer***. The inputs are directly fed into the outputs via a series of weights. It's a **[feed-forward network](https://en.wikipedia.org/wiki/Feedforward_neural_network)** where the information moves in only one direction, i.e. forward direction from input nodes to output nodes.\n", "\n", "A **multi-layer perceptron model** is the other kind of neural network where there are one or more hidden layers in between input and output layers. The information flows from input layer to hidden layers and then to output layers. These models can be of **feed-forward** type or they can also use **[back-propagation](https://en.wikipedia.org/wiki/Backpropagation)** method. In back-propagation, the error is calculated in the output layer by computing the difference of actual output and predicted output. The error is then distributed back to the network layers. Based on this error, the algorithm will adjust the weights of each connection in order to reduce the error value. This type of learning is also referred as **deep learning**.\n", "\n", "We create a **simple neural network** model with **one hidden layer** with 784 neurons. Our input layer will also have 784 neurons as we have flattened out training dataset into a single 784 dimensional vector.\n", "\n", "*softmax* activation is used in the output layer.\n", "\n", "*adam* gradient descent optimizer is used to learn weights."]}, {"cell_type": "code", "metadata": {"_uuid": "8680fa418a3505fbd2055f3a737201dc920b4729", "collapsed": true, "_cell_guid": "7186018f-1100-4333-84f0-b9f3fdf65ae7"}, "execution_count": null, "source": ["def baseline_model():\n", "    # create model\n", "    model = Sequential()\n", "    model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))\n", "    model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))\n", "    # compile model\n", "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n", "    return model"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "75cda5df5bee9ffd14517c89a43614ba368b2ead", "_cell_guid": "608576c4-5c6e-40ba-9499-7f9a2ec44b72"}, "source": ["## Fit and Evaluate Model\n", "\n", "The model is fit over 5 epochs/iteration. It takes a batch of 200 images in each iteration. Validation dataset is used for validation. The epochs may be increased to improve accuracy.\n", "\n", "Finally, validation dataset is used to evaluate the model by calculating the model's classification accuracy."]}, {"cell_type": "code", "metadata": {"_uuid": "16906a32f6499e74d853b93b31061e20e14dcdf9", "_cell_guid": "6e0a2d8b-55f9-49d7-9204-f9ed701edf47"}, "execution_count": null, "source": ["model = baseline_model()\n", "model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=5, batch_size=200, verbose=1)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "619fa1512e5d345059d9447d18be5c8786b6d431", "_cell_guid": "ed94ee5d-a61f-411d-841b-c29762f6bba1"}, "execution_count": null, "source": ["model.summary()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "65b9373ba459c42fd2ad8c376e4f7cc1d839d756", "_cell_guid": "a618a6fb-976b-42f2-8b5e-e997b301c28c"}, "execution_count": null, "source": ["scores = model.evaluate(X_val, Y_val, verbose=0)\n", "print (scores)\n", "print ('Score: {}'.format(scores[0]))\n", "print ('Accuracy: {}'.format(scores[1]))"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "56414218ede10949566d7ab33835abb351e4b6fe", "_cell_guid": "7c24f83a-1f9d-4d89-9748-35e018fcae07"}, "source": ["## Plot correctly and incorrectly predicted images\n", "\n", "Let's plot some images which are correctly predicted and some images which are incorrectly predicted on our validation dataset."]}, {"cell_type": "code", "metadata": {"_uuid": "2a822dfea453ccb1301058aac38811052b660cfc", "_cell_guid": "c8465312-8cad-4455-aa5e-93996c7e0c29"}, "execution_count": null, "source": ["# get predicted values\n", "predicted_classes = model.predict_classes(X_val)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "8ebe4a42f4623b331712623d901a0683f42d3392", "collapsed": true, "_cell_guid": "6df81428-7752-4344-9e5e-5e881abb5c6d"}, "execution_count": null, "source": ["# get index list of all correctly predicted values\n", "correct_indices = np.nonzero(np.equal(predicted_classes, y_val))[0]\n", "\n", "# get index list of all incorrectly predicted values\n", "incorrect_indices = np.nonzero(np.not_equal(predicted_classes, y_val))[0]"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "81bbfa9148ac0095329c4205a798e6555e61b2c2", "_cell_guid": "616811c3-f465-4115-887e-09d3f2f89cc2"}, "execution_count": null, "source": ["print ('Correctly predicted: %i' % np.size(correct_indices))\n", "print ('Incorrectly predicted: %i' % np.size(incorrect_indices))"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_uuid": "5cf066897328a30306005950bac146d5ed47467d", "_cell_guid": "2e43ccdb-85d5-48eb-847a-8756a7f97bfc"}, "execution_count": null, "source": ["plt.figure(figsize=[20,8])\n", "for i, correct in enumerate(correct_indices[:6]):\n", "    plt.subplot(1,6,i+1)\n", "    plt.imshow(X_val[correct].reshape(28,28), cmap='gray', interpolation='none')\n", "    plt.title(\"Predicted {}, Class {}\".format(predicted_classes[correct], y_val[correct]))\n", "    \n", "plt.figure(figsize=[20,8])\n", "for i, incorrect in enumerate(incorrect_indices[:6]):\n", "    plt.subplot(1,6,i+1)\n", "    plt.imshow(X_val[incorrect].reshape(28,28), cmap='gray', interpolation='none')\n", "    plt.title(\"Predicted {}, Class {}\".format(predicted_classes[incorrect], y_val[incorrect]))"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "833343e281aa740476fa6dfcfa31b3d37e4ed6b8", "_cell_guid": "828fbc21-46f2-4f23-9095-3deebe7a7c49"}, "source": ["## Confusion Matrix"]}, {"cell_type": "code", "metadata": {"_uuid": "88b715d13561d4fe262fccdd791b3f8858d83808", "_cell_guid": "ab4e1116-1606-453f-a8c2-9d2cfb8688d2"}, "execution_count": null, "source": ["# we have digit labels from 0 to 9\n", "# we can either manually create a class variable with those labels\n", "# class_names = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n", "\n", "# or, we can take unique values from train dataset's labels\n", "class_names = np.unique(y_train)\n", "\n", "# Compute confusion matrix\n", "cnf_matrix = confusion_matrix(y_val, predicted_classes)\n", "np.set_printoptions(precision=2)\n", "\n", "print ('Confusion Matrix in Numbers')\n", "print (cnf_matrix)\n", "print ('')\n", "\n", "cnf_matrix_percent = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]\n", "\n", "print ('Confusion Matrix in Percentage')\n", "print (cnf_matrix_percent)\n", "print ('')\n", "\n", "true_class_names = class_names\n", "predicted_class_names = class_names\n", "\n", "df_cnf_matrix = pd.DataFrame(cnf_matrix, \n", "                             index = true_class_names,\n", "                             columns = predicted_class_names)\n", "\n", "df_cnf_matrix_percent = pd.DataFrame(cnf_matrix_percent, \n", "                                     index = true_class_names,\n", "                                     columns = predicted_class_names)\n", "\n", "plt.figure(figsize = (8,6))\n", "\n", "#plt.subplot(121)\n", "ax = sns.heatmap(df_cnf_matrix, annot=True, fmt='d')\n", "ax.set_ylabel('True values')\n", "ax.set_xlabel('Predicted values')\n", "ax.set_title('Confusion Matrix in Numbers')\n", "\n", "'''\n", "plt.subplot(122)\n", "ax = sns.heatmap(df_cnf_matrix_percent, annot=True)\n", "ax.set_ylabel('True values')\n", "ax.set_xlabel('Predicted values')\n", "'''"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "a10cd9083a3d270d9575ca583cf89e34fae665f0", "_cell_guid": "b6fc6d01-3f61-41eb-922f-911f089fa81f"}, "source": ["Out of 4200 validation data, we got the following correct and incorrect predictions by MLP model:\n", "- Correctly predicted: 4109\n", "- Incorrectly predicted: 91\n", "\n", "The above confusion matrix heatmap shows that:\n", "- 9 values of digit 9 were predicted as 7.\n", "- 6 values of digit 3 were perdicted as 8.\n", "- 4 values of digit 4, 5 and 9 were predicted as 9, 3 and 4 respectively.\n", "\n", "The accuracy of the model may improve if we increase the epoch/iteration number while fitting the model. Currently, it is set as 5. We can increase it to 10 and see the accuracy output."]}, {"cell_type": "markdown", "metadata": {"_uuid": "89e7480133876fd0b9492f0ffd6505f43b4fafc6", "_cell_guid": "8ff82e92-52db-421b-b464-bc9b39dc8cf4"}, "source": ["## Improve Accuracy using Convolution Neural Network (CNN) Model\n", "\n", "[Convolutional Neural Networks (CNN)](https://en.wikipedia.org/wiki/Convolutional_neural_network) are similar to Multi-layer Perceptron Neural Networks. They are also made up of neurons that have learnable weights and biases. CNNs have been successfully applied to analyzing visual imagery. They are mostly being applied in image and video recognition, recommender systems and natural language processing.\n", "\n", "A CNN consists of multiple hidden layers. The hidden layers are either *convolutional*, *pooling* or *fully connected*.\n", "\n", "**Convolution layer:** Feature extraction is done in this layer. This layer applies convolution operation to the input and pass the result to the next layer. In the image classification problem, a weight matrix is defined in the convolution layer. A dot product is computed between the weight matrix and a small part (as the size of the weight matrix) of the input image. The weight runs across the image such that all the pixels are covered at least once, to give a convolved output.\n", "\n", "> The weight matrix behaves like a **filter** in an image extracting particular information from the original image matrix. \n", ">\n", ">A weight combination might be extracting edges, while another one might a particular color, while another one might just blur the unwanted noise.\n", ">\n", ">The weights are learnt such that the loss function is minimized similar to a Multi-layer Perceptron. \n", ">\n", ">Therefore weights are learnt to extract features from the original image which help the network in correct prediction. \n", ">\n", ">When we have multiple convolutional layers, the initial layer extract more generic features, while as the network gets deeper, the features extracted by the weight matrices are more and more complex and more suited to the problem at hand.\n", ">\n", ">*Reference: [Architecture of Convolutional Neural Networks (CNNs) demystified](https://www.analyticsvidhya.com/blog/2017/06/architecture-of-convolutional-neural-networks-simplified-demystified/)*\n", "\n", "***Stride:*** While computing the dot product, if the weight matrix moves 1 pixel at a time then we call it a stride of 1. Size of the image keeps on reducing as we increase the stride value.\n", "\n", "***Padding:*** Padding one or more layer of zeros across the image helps to resolve the output image size reduction issue caused by *stride*. Initial size of the image is retained after the padding is done.\n", "\n", "**Pooling layer:** Reduction in number of feature parameters is done in this layer. When the image size is too larger, then we need a pooling layer in-between two convolution layers. This layer helps to reduce the number of trainable parameters of the input image. The sole purpose of pooling is to reduce the spatial size of the image. This layer is also used to control overfitting. \n", "- Max pooling: Uses maximum value from each of the cluster of the prior layer\n", "- Average pooling: Uses the average value from each of the cluster of the prior layer\n", "\n", "**Fully connected layer:** This layer comes after convolution and pooling layers. This layer connects each neuron in one layer to every neuron in another layer. This is similar to the concept of layer connection of Multi-layer perceptron model. Error is computed in the output layer by computing the difference in actual output and predicted output. After that, back-propagation is used to update the weight and biases for error and loss reduction."]}, {"cell_type": "markdown", "metadata": {"_uuid": "5c4f44230c2c634f8f645406c586763dd35d1ff4", "_cell_guid": "3e401476-30ec-46a1-a2bc-15f4614273ab"}, "source": ["## Load train and test data\n", "\n", "Let's again load the train and test datasets."]}, {"cell_type": "code", "metadata": {"_uuid": "bb767f75e76a2281c1114c15c480f15d854832a8", "_cell_guid": "62ae8b05-6835-4f15-bc2f-493c1298953f"}, "execution_count": null, "source": ["train = pd.read_csv('../input/train.csv')\n", "print (train.shape)\n", "train.head()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "e7a29bfe4a5b973a7b7d159a4e4bec982115eb31", "_cell_guid": "4af9eae7-a723-4b85-8134-4e02ac93f2d0"}, "execution_count": null, "source": ["test = pd.read_csv('../input/test.csv')\n", "print (test.shape)\n", "test.head()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "cd0be178e7a318b68df16f1d3d85dd0e93cc7ed0", "collapsed": true, "_cell_guid": "6eeaa459-e192-494b-abec-c1ae5ddb32f5"}, "execution_count": null, "source": ["y_train = train['label']\n", "X_train = train.drop(labels=['label'], axis=1)\n", "X_test = test"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "c58ee8614b55a0c79401f42a37ebbe2b19872508", "_cell_guid": "82a61cde-1ea3-447b-9357-709b4fd8a2c5"}, "source": ["## Get values of data"]}, {"cell_type": "code", "metadata": {"_uuid": "c257f80f3c55e3a0d4d4cfb878dc6dc6ca952223", "collapsed": true, "_cell_guid": "741ebb46-95e9-4ac5-b963-1c2571d1df22"}, "execution_count": null, "source": ["X_train = X_train.values.astype('float32') # pixel values of all images in train set\n", "y_train = y_train.values.astype('int32') # labels of all images\n", "X_test = test.values.astype('float32') # pixel values of all images in test set"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "74409e4b9990a13753663ba5b25c80101b60b0ea", "_cell_guid": "015b5489-48e7-4220-a1c9-780fdf494597"}, "source": ["## View shape and content of data"]}, {"cell_type": "code", "metadata": {"_uuid": "46fc16e013aedcf40b012a234f396da1be479de0", "_cell_guid": "8be6dfd7-928f-4924-8cbd-97ecb2aef096"}, "execution_count": null, "source": ["print (X_train.shape)\n", "print (y_train.shape)\n", "print (X_train[1])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "87b05ce0893878f22acd498214e88d2a8d40441e", "_cell_guid": "34c5f268-fcca-4ba8-b24a-acc64df65407"}, "source": ["## Normalizing input values\n", "\n", "As we can see above, the pixel values for each image are gray scaled between 0 and 255. We now, normalize those values from 0-255 to 0-1."]}, {"cell_type": "code", "metadata": {"_uuid": "5cf09a46c86345211769c9bd759c41b43095fc32", "_cell_guid": "974a5bdb-0a80-48b7-9d55-fd502e705986"}, "execution_count": null, "source": ["# pixel values are gray scale between 0 and 255\n", "# normalize inputs from 0-255 to 0-1\n", "X_train = X_train / 255\n", "X_test = X_test / 255\n", "print (X_train[1])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "fdb4f7e724d0784ac65702ede34f86e49fecdea0", "_cell_guid": "d7b498a1-c9c6-41a4-97b4-dae483b69a29"}, "source": ["## Converting target variable values into one-hot format\n", "\n", "The output/target variable is in the format 0 to 9. As this is a multi-class classification problem, we convert the output class values into one-hot format which is simply a binary matrix, i.e.\n", "\n", ">value 0 will be converted to one-hot format as [1, 0, 0, 0, 0, 0, 0, 0, 0]\n", ">\n", ">value 1 will be converted to one-hot format as [0, 1, 0, 0, 0, 0, 0, 0, 0]\n", ">\n", ">value 2 will be converted to one-hot format as [0, 0, 1, 0, 0, 0, 0, 0, 0]\n", ">\n", ">and so on...\n"]}, {"cell_type": "code", "metadata": {"_uuid": "b1e692dd76ae9c5d62a12ad4f8c8a2d78d4c34ad", "_cell_guid": "ed4434c9-5a11-4b4a-bf0e-6c83bbf125fa"}, "execution_count": null, "source": ["print (y_train.shape)\n", "print (y_train[0])"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "2dc85df66195edec05556af7929fdaa9a1e607a2", "collapsed": true, "_cell_guid": "800c0dbe-a827-4cfe-8ef7-9bcca944188b"}, "execution_count": null, "source": ["# one hot encode outputs\n", "# note that we have new variables with capital Y\n", "# Y_train is different than y_train\n", "Y_train = np_utils.to_categorical(y_train)\n", "num_classes = Y_train.shape[1]"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "d9c99be3468d4850fef4a05f99679b4bc0b7b8c6", "_cell_guid": "0b65a4de-6092-4be6-a51f-18f756c28903"}, "execution_count": null, "source": ["print (y_train.shape, Y_train.shape)\n", "print (y_train[0], Y_train[0])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "7c605696e38621ef1e153f8615d526b59c5f8522", "_cell_guid": "18c8b5ef-f911-43b5-9cce-e562680a6f44"}, "source": ["## Splitting train dataset into training and validation set\n", "\n", "We split the train dataset into two parts in 9:1 ratio. 90% will be the actual training set and the remaining 10% will be the validation/testing set. We train our model using the training set and test the accuracy of the model using the validation set."]}, {"cell_type": "code", "metadata": {"_uuid": "c333574cc5c251b54b4d6568056ebdf448c37116", "collapsed": true, "_cell_guid": "1162f8ef-5658-48ae-a5c8-431dbf6f708b"}, "execution_count": null, "source": ["# Split the entire training set into two separate sets: Training set and Validation set\n", "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.10, random_state=random_seed)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "7773b2da15ec10ba47258cdbd5cb3639bfbc3525", "_cell_guid": "2b903687-b94c-4e84-a864-00d2d3ef985a"}, "execution_count": null, "source": ["print (X_train.shape, Y_train.shape, X_val.shape, Y_val.shape)\n", "num_pixels = X_train.shape[1]"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "f84ce1674dabff45f06bc9a48f1e74a9a636f5ef", "_cell_guid": "25be84f5-ed9d-43a6-b6e9-cd2a924d62c7"}, "execution_count": null, "source": ["print (Y_val)\n", "# converting one-hot format of digits to normal values/labels\n", "y_val = np.argmax(Y_val, 1) # reverse of to_categorical\n", "print (y_val)\n", "# Note that: capital Y_val contains values in one-hot format and small y_val contains normal digit values"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "2c2498e7b1204a5281e2aba7254e8cac47c80b19", "_cell_guid": "ba98589b-eeea-4b91-8caa-b7a125da83d8"}, "source": ["## Reshaping images\n", "\n", "The image dimension expected by Keras for 2D (two-dimensional) convolution is in the format of **[pixels][width][height]**.\n", "\n", "For RGB color image, the first dimension (pixel) value would be 3 for the red, green and blue components. It's like having 3 image inputs for every single color image. In our case (for MNIST handwritten images), we have gray scale images. Hence, the pixel dimension is set as 1."]}, {"cell_type": "code", "metadata": {"_uuid": "faa9fc70dfa86f6b493cd2cac5693f22a2dbbbfd", "_cell_guid": "697de529-d879-44d3-9f39-5a2932e1877c"}, "execution_count": null, "source": ["# reshape to be [samples][pixels][width][height]\n", "X_train = X_train.reshape(X_train.shape[0], 1, 28, 28).astype('float32')\n", "X_test = X_test.reshape(X_test.shape[0], 1, 28, 28).astype('float32')\n", "X_val = X_val.reshape(X_val.shape[0], 1, 28, 28).astype('float32')\n", "\n", "print (num_pixels, X_train.shape, X_test.shape, X_val.shape)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "bd4c7b3ab52a8a3b4b2940b966031874c55dca25", "_cell_guid": "f882fc3e-bb86-4f61-803d-95f236096bcb"}, "execution_count": null, "source": ["print (X_train[1])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "43a532df5edb9172ff6361d5ea50bed3c467fae9", "_cell_guid": "3c12a745-04af-4a67-acf7-ec9f5ec91b83"}, "source": ["## Define Convolutional Neural Network (CNN) Model\n", "\n", "**Convolution Layer**\n", "- We define 32 feature maps with the size of 5x5 matrix\n", "- We use ReLU (Rectified Linear Units) as the activation function\n", "- This layer expects input image size of 1x28x28 ([pixels][height][weight])\n", "\n", "**Max Pooling Layer**\n", "- It has a pool size of 2x2\n", "\n", "**Dropout Layer**\n", "- Configured to randomly exclude 20% of neurons in the layer to reduce overfitting\n", "\n", "**Flatten**\n", "- Flattens the image into a single dimensional vector which is required as input by the fully connected layer\n", "\n", "**Fully connected Layer**\n", "- Contains 128 neurons\n", "- relu is used as an activation function\n", "- Output layer has num_classes=10 neurons for the 10 classes\n", "- softmax activation function is used in the output layer\n", "- adam is used as optimizer to learn and update weights"]}, {"cell_type": "code", "metadata": {"_uuid": "6f06976cccb8f64a8e594ccf6d634d8707ea281e", "collapsed": true, "_cell_guid": "c01a29ad-2af7-4ad7-8b86-df0f28d35add"}, "execution_count": null, "source": ["# baseline model for CNN\n", "def baseline_model():\n", "    # create model    \n", "    model = Sequential()    \n", "    model.add(Conv2D(32, (5, 5), input_shape=(1, 28, 28), activation='relu'))    \n", "    model.add(MaxPooling2D(pool_size=(2, 2)))    \n", "    model.add(Dropout(0.2))\n", "    model.add(Flatten())    \n", "    model.add(Dense(128, activation='relu'))\n", "    model.add(Dense(num_classes, activation='softmax'))    \n", "    # compile model\n", "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])    \n", "    return model"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "22617c39374566467fc274b2551613741ec9fe59", "_cell_guid": "d6bca10f-ef15-419f-acf7-efeb43a61fff"}, "source": ["To compile the model, there are [different optimizers](https://keras.io/optimizers/) present in Keras like Stochastic Gradient Descent optimizer, Adam optimizer, RMSprop optimizer, etc."]}, {"cell_type": "code", "metadata": {"_uuid": "7f057261d3f5f3d3c245250d32e9187796e89ec2", "collapsed": true, "_cell_guid": "c8093c62-c7dc-422f-9edd-ebff9c49ca8e"}, "execution_count": null, "source": ["# Example of using RMSprop optimizer\n", "#from keras.optimizers import RMSprop, SGD\n", "#model.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=0.001), metrics=['accuracy'])\n", "#model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.001), metrics=['accuracy'])"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "2c39271bf6091cd13136149233a04a4d0e02055c", "_cell_guid": "3eea888a-1c94-42c6-b598-a85b7da218c4"}, "source": ["## Fit and Evaluate Model\n", "\n", "The model is fit over 5 epochs/iteration. It takes a batch of 200 images in each iteration. Validation data is used as validation set. The epochs may be increased to improve accuracy.\n", "\n", "Finally, validation data is used to evaluate the model by calculating the model's classification accuracy."]}, {"cell_type": "code", "metadata": {"_uuid": "2fa166f59331712b515593ac7d1068b5caccb786", "_cell_guid": "7786a45b-cd56-4317-94c9-8c318645a565"}, "execution_count": null, "source": ["model = baseline_model()\n", "history = model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=5, batch_size=200, verbose=1)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "b8d51768527637bda19be880f4caca8f4a31e2bb", "_cell_guid": "3b75d1ca-8184-4a49-9f43-80e92be703e9"}, "execution_count": null, "source": ["history_dict = history.history\n", "history_dict.keys()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "32387c87f970b0d96da52fe56818411caba6229f", "_cell_guid": "26d46739-e496-4eeb-9827-0ab1de91a7c0"}, "execution_count": null, "source": ["plt.figure(figsize=[10,4])\n", "\n", "plt.subplot(121)\n", "plt.plot(range(1, len(history_dict['val_acc'])+1), history_dict['val_acc'])\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Accuracy')\n", "\n", "plt.subplot(122)\n", "plt.plot(range(1, len(history_dict['val_loss'])+1), history_dict['val_loss'])\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Loss')"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "99c2b604b9ac3b1289d583a40c504a3c355083cb", "_cell_guid": "2f3de145-2579-4995-a5fc-ff2c2e831053"}, "execution_count": null, "source": ["model.summary()"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_uuid": "10ebd20e2b17af856a4ca5c08867b7346892882a", "_cell_guid": "d22eace8-6393-44a6-b2c0-f50bab3cb10f"}, "execution_count": null, "source": ["scores = model.evaluate(X_val, Y_val, verbose=0)\n", "print (scores)\n", "print ('Score: {}'.format(scores[0]))\n", "print ('Accuracy: {}'.format(scores[1]))"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "cdd0470942b41a5e1b40b6658173f22265162a6b", "_cell_guid": "f11a6fac-9c62-4d9f-988d-e435b5deaff9"}, "source": ["Accuracy (98.64%) of Convolution Neural Network (CNN) model has improved as compared to the accuracy (97.83%) of Multi-layer Perceptron (MLP) model. \n", "\n", "The accuracy of CNN model can be further increased by:\n", "- increasing the epoch number while fitting the model\n", "- adding more convolution and pooling layers to the model"]}, {"cell_type": "markdown", "metadata": {"_uuid": "ba3b2cacdaea1d64b9fd2acb693f7783c7770932", "_cell_guid": "ce148b2d-e1d3-45f3-b99c-ae8c4ae6c8e6"}, "source": ["## Plot correctly and incorrectly predicted images\n", "\n", "Let's plot some images which are correctly predicted and some images which are incorrectly predicted on our test dataset."]}, {"cell_type": "code", "metadata": {"scrolled": true, "_uuid": "607be7d258df18c18e84b8993cb47d03ec62998b", "_cell_guid": "09074a1a-39b0-4d81-a516-6b4a8ae404ae"}, "execution_count": null, "source": ["# get predicted values\n", "predicted_classes = model.predict_classes(X_val)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "f85d25b0395583cec53a3037baf7b61fb4e93128", "collapsed": true, "_cell_guid": "7689368c-2e9a-430f-a36e-a06dae0919d2"}, "execution_count": null, "source": ["# get index list of all correctly predicted values\n", "correct_indices = np.nonzero(np.equal(predicted_classes, y_val))[0]\n", "\n", "# get index list of all incorrectly predicted values\n", "incorrect_indices = np.nonzero(np.not_equal(predicted_classes, y_val))[0]"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "7f432f7dc778d3acbee8ff7799f6d86800bddaee", "_cell_guid": "9a4a7007-dcfa-44a2-bb32-8aeada4c9bb3"}, "execution_count": null, "source": ["print ('Correctly predicted: %i' % np.size(correct_indices))\n", "print ('Incorrectly predicted: %i' % np.size(incorrect_indices))"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_uuid": "98529ec9e775e6471d9a72506c8723cc6dcaaab6", "_cell_guid": "8d29a47c-f699-4312-bdc3-fae27148709b"}, "execution_count": null, "source": ["plt.figure(figsize=[20,8])\n", "for i, correct in enumerate(correct_indices[:6]):\n", "    plt.subplot(1,6,i+1)\n", "    plt.imshow(X_val[correct].reshape(28,28), cmap='gray', interpolation='none')\n", "    plt.title(\"Predicted {}, Class {}\".format(predicted_classes[correct], y_val[correct]))\n", "    \n", "plt.figure(figsize=[20,8])\n", "for i, incorrect in enumerate(incorrect_indices[:6]):\n", "    plt.subplot(1,6,i+1)\n", "    plt.imshow(X_val[incorrect].reshape(28,28), cmap='gray', interpolation='none')\n", "    plt.title(\"Predicted {}, Class {}\".format(predicted_classes[incorrect], y_val[incorrect]))"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "de16a86f7ee09f68b1f818fbb179b5eaa0884123", "_cell_guid": "97058a78-d3f5-4888-89a6-6ca1175bb923"}, "source": ["## Confusion Matrix"]}, {"cell_type": "code", "metadata": {"_uuid": "c78985215e09e5b2d4d923cfefe8faebd98de488", "_cell_guid": "ad0e2c5a-def2-4310-8ccc-caaf1553d7f5"}, "execution_count": null, "source": ["# we have digit labels from 0 to 9\n", "# we can either manually create a class variable with those labels\n", "# class_names = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n", "\n", "# or, we can take unique values from train dataset's labels\n", "class_names = np.unique(y_train)\n", "\n", "# Compute confusion matrix\n", "cnf_matrix = confusion_matrix(y_val, predicted_classes)\n", "np.set_printoptions(precision=2)\n", "\n", "print ('Confusion Matrix in Numbers')\n", "print (cnf_matrix)\n", "print ('')\n", "\n", "cnf_matrix_percent = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]\n", "\n", "print ('Confusion Matrix in Percentage')\n", "print (cnf_matrix_percent)\n", "print ('')\n", "\n", "true_class_names = class_names\n", "predicted_class_names = class_names\n", "\n", "df_cnf_matrix = pd.DataFrame(cnf_matrix, \n", "                             index = true_class_names,\n", "                             columns = predicted_class_names)\n", "\n", "df_cnf_matrix_percent = pd.DataFrame(cnf_matrix_percent, \n", "                                     index = true_class_names,\n", "                                     columns = predicted_class_names)\n", "\n", "plt.figure(figsize = (8,6))\n", "\n", "#plt.subplot(121)\n", "ax = sns.heatmap(df_cnf_matrix, annot=True, fmt='d')\n", "ax.set_ylabel('True values')\n", "ax.set_xlabel('Predicted values')\n", "ax.set_title('Confusion Matrix in Numbers')\n", "\n", "'''\n", "plt.subplot(122)\n", "ax = sns.heatmap(df_cnf_matrix_percent, annot=True)\n", "ax.set_ylabel('True values')\n", "ax.set_xlabel('Predicted values')\n", "'''"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "50ef2b5bec55e59543662d353481e07ef537139f", "_cell_guid": "7c0fd284-08a8-476d-ae11-03901a33230a"}, "source": ["### MLP Model outcome\n", "\n", "Out of 4200 validation data, we had got the following correct and incorrect predictions by MLP model:\n", "- Correctly predicted: 4109\n", "- Incorrectly predicted: 91\n", "\n", "Using Multi-layer Perceptron (MLP) Model, we had the following heatmap outcome:\n", "- 9 values of digit 9 were predicted as 7.\n", "- 6 values of digit 3 were perdicted as 8.\n", "- 4 values of digit 4, 5 and 9 were predicted as 9, 3 and 4 respectively.\n", "\n", "### CNN Model outcome\n", "\n", "Out of 4200 validation data, we had got the following correct and incorrect predictions by CNN model:\n", "- Correctly predicted: 4146\n", "- Incorrectly predicted: 57\n", "\n", "Using Convolutional Neural Network (CNN) Model, we had the following improvements:\n", "- Number 9 predicted as 7 has been reduced from 9 to 7 times.\n", "- Number 3 predicted as 8 has been reduced from 6 to 1 times.\n", "- Number 4,9 predicted as 9,4 repectively has been reduced from 4 times to 2 times.\n", "\n", "The accuracy of CNN model can be further increased by:\n", "- increasing the epoch/iteration number while fitting the model\n", "- adding more convolution and pooling layers to the model"]}, {"cell_type": "markdown", "metadata": {"_uuid": "05b4c5cc2f77e4637663930069eff789b49930c4", "_cell_guid": "f0d779ca-1433-4e50-829d-7779f25d1e6d"}, "source": ["## Improving accuracy using multiple CNN layer\n", "\n", "Let's try adding multiple convolution layers (*Conv2D*) and multiple fully-connected layers (*Dense*) as well.\n", "\n", ">The second Convolution layer will have 15 filters with the size of 3x3 matrix.\n", ">\n", ">The second fully-connected layer will have 50 neurons.\n", "\n", "We also use 10 epochs this time instead of 5."]}, {"cell_type": "code", "metadata": {"_uuid": "b0d1db17b11e96841961287539d9eff48e21d719", "collapsed": true, "_cell_guid": "413aec46-207b-48c9-8ce9-6773383c5343"}, "execution_count": null, "source": ["def baseline_model():\n", "    # create model\n", "    model = Sequential()\n", "    \n", "    model.add(Conv2D(filters=30, kernel_size=(5, 5), input_shape=(1, 28, 28), activation='relu'))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    model.add(Dropout(0.2))\n", "    \n", "    model.add(Conv2D(filters=15, kernel_size=(3, 3), activation='relu'))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    model.add(Dropout(0.2))\n", "    \n", "    model.add(Flatten())\n", "    \n", "    model.add(Dense(512, activation='relu'))\n", "    model.add(Dropout(0.25))\n", "    \n", "    model.add(Dense(1024, activation='relu'))\n", "    model.add(Dropout(0.5))\n", "    \n", "    model.add(Dense(num_classes, activation='softmax'))\n", "    \n", "    # Compile model\n", "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n", "    \n", "    return model"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": false, "_uuid": "ec37517208211719d9e3547d9649a4735f8fa5ea", "_cell_guid": "9b8dbc09-3406-4cf6-83e4-4ed523ab022f"}, "execution_count": null, "source": ["# build the model\n", "model = baseline_model()\n", "\n", "# fit the model\n", "history = model.fit(X_train, Y_train, validation_data=(X_val, Y_val), epochs=10, batch_size=200)"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "61be5ab8d0d777e299a5c83420aba213837f9bb1", "_cell_guid": "e572caf5-675d-4fe8-a42f-e2a3b3acf001"}, "execution_count": null, "source": ["history_dict = history.history\n", "history_dict.keys()"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "17d504f0fc752ae21bdf580faceec147969bac78", "_cell_guid": "c691be6a-4f63-4fe0-abcb-28ac52257710"}, "execution_count": null, "source": ["plt.figure(figsize=[10,4])\n", "\n", "plt.subplot(121)\n", "plt.plot(range(1, len(history_dict['val_acc'])+1), history_dict['val_acc'])\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Accuracy')\n", "\n", "plt.subplot(122)\n", "plt.plot(range(1, len(history_dict['val_loss'])+1), history_dict['val_loss'])\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Loss')"], "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "89e954dea59bb35d7b4b9d5477c62e71d3c89701", "_cell_guid": "3629ef8e-b098-4fed-a091-fe7162c8d976"}, "execution_count": null, "source": ["model.summary()"], "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": false, "_uuid": "89dcfd2fa3261ea2f9925cd7ce65a14fad8621ea", "_cell_guid": "16a4c478-99fc-4e2f-9708-58e152e7aeef"}, "execution_count": null, "source": ["scores = model.evaluate(X_val, Y_val, verbose=0)\n", "print (scores)\n", "print ('Score: {}'.format(scores[0]))\n", "print ('Accuracy: {}'.format(scores[1]))"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "4400f720ea00f092c05c94455ff84a54635bb0d2", "_cell_guid": "7454ed5e-07e1-443b-ab70-c2519638b2d3"}, "source": ["Accuracy has improved from **98.64%** to **99.00%**."]}, {"cell_type": "markdown", "metadata": {"_uuid": "74d62af0ed063f6030e203e58e5d67517ba0534b", "_cell_guid": "b2b61f5d-1ce0-4033-b779-56579fc20e3f"}, "source": ["## Submission to Kaggle"]}, {"cell_type": "code", "metadata": {"_uuid": "7c741b2e536b130d6c6a3b8ff031516b2ce05f10", "_cell_guid": "6bf6661a-a69b-44cc-af77-014cafa80285"}, "execution_count": null, "source": ["# get predicted values for test dataset\n", "predicted_classes = model.predict_classes(X_test)\n", "\n", "submissions = pd.DataFrame({'ImageId': list(range(1, len(predicted_classes) + 1)), \n", "                            \"Label\": predicted_classes})\n", "\n", "#submissions.to_csv(\"submission.csv\", index=False, header=True)"], "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "be95232346c9043898af94926b17387e2c2037bf", "_cell_guid": "743c6f9d-e6a3-4380-8eee-4a4597082483"}, "source": ["## References\n", "\n", "- [Handwritten Digit Recognition using Convolutional Neural Networks in Python with Keras](https://machinelearningmastery.com/handwritten-digit-recognition-using-convolutional-neural-networks-python-keras/)\n", "- [Building a simple neural-network with Keras](https://github.com/wxs/keras-mnist-tutorial/blob/master/MNIST%20in%20Keras.ipynb)\n", "- [Architecture of Convolutional Neural Networks (CNNs) demystified](https://www.analyticsvidhya.com/blog/2017/06/architecture-of-convolutional-neural-networks-simplified-demystified/)\n", "- [Welcome to deep learning (CNN 99%)](https://www.kaggle.com/toregil/welcome-to-deep-learning-cnn-99)\n", "- [Introduction to CNN Keras - 0.997 (top 6%)](https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6)\n"]}], "metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"file_extension": ".py", "mimetype": "text/x-python", "name": "python", "pygments_lexer": "ipython3", "version": "3.6.1", "codemirror_mode": {"version": 3, "name": "ipython"}, "nbconvert_exporter": "python"}}, "nbformat_minor": 1}