{"cells":[{"metadata":{"id":"zQHYkAoVUMKi","colab_type":"text"},"cell_type":"markdown","source":"## 1. Importing Library"},{"metadata":{"id":"LYlQnHuARMFQ","colab_type":"code","outputId":"53fccd35-7077-46f4-d7f5-3e8d77d18508","colab":{"base_uri":"https://localhost:8080/","height":51},"trusted":true},"cell_type":"code","source":"# Importing Tensorflow and keras\n#Keras is built into TF 2.0\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as img\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\n#Setting the Theme of the data visualizer Seaborn\nsns.set(style=\"white\",context=\"notebook\",palette=\"deep\")\n\n#Tensorflow Version\nprint(tf.version.VERSION)\nprint(tf.keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{"id":"WEPDqrJLUWEi","colab_type":"text"},"cell_type":"markdown","source":"##2.Data Preparation"},{"metadata":{"id":"UutK5S_rUGmH","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"id":"05yoL078UJIp","colab_type":"code","outputId":"87e71ffe-2da5-4e74-d1e9-c66cc4061b84","colab":{"base_uri":"https://localhost:8080/","height":478},"trusted":true},"cell_type":"code","source":"Y_train = train[\"label\"]\n\n#Dropping Label Column\nX_train = train.drop(labels=[\"label\"],axis=1)\n\n#free up some space\ndel train\n\ngraph = sns.countplot(Y_train)\n \nY_train.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"id":"jGSazKXLXRtA","colab_type":"code","outputId":"5c6958f8-b12c-44d5-f0e3-dc7eb1546c1e","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":true},"cell_type":"code","source":"#Checking for any null or missing values\nX_train.isnull().any().describe()\n\ntest.isnull().any().describe()\n","execution_count":null,"outputs":[]},{"metadata":{"id":"xJQsCTooXv1X","colab_type":"text"},"cell_type":"markdown","source":"###Normalisation"},{"metadata":{"id":"pTFl1nLqXpSy","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"X_train = X_train/255\ntest = test/255","execution_count":null,"outputs":[]},{"metadata":{"id":"0Lx2A8K8YAKn","colab_type":"text"},"cell_type":"markdown","source":"###Reshape"},{"metadata":{"id":"RgJra9a4X9U1","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"id":"GSbzJTHlYkhS","colab_type":"text"},"cell_type":"markdown","source":"### Label Encoding"},{"metadata":{"id":"U7DvS95KYrVa","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"Y_train = tf.keras.utils.to_categorical(Y_train, num_classes=10)\n#To enable label into hot vector. For Eg.7 -> [0,0,0,0,0,0,0,1,0,0]","execution_count":null,"outputs":[]},{"metadata":{"id":"HUz1lMAAZLFC","colab_type":"code","colab":{},"trusted":true},"cell_type":"markdown","source":"#Spliting Train and test set"},{"metadata":{"id":"UUBksBY12L8V","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"X_train,X_val,Y_train,Y_val = train_test_split(X_train,Y_train,test_size=0.1,\n                                                random_state = 2)","execution_count":null,"outputs":[]},{"metadata":{"id":"KVNgcZ5F3Ap5","colab_type":"code","outputId":"4e5029cb-8519-42c8-f57f-12d7f35e8eb3","colab":{"base_uri":"https://localhost:8080/","height":274},"trusted":true},"cell_type":"code","source":"#Show some example \n\ng = plt.imshow(X_train[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"id":"tU1XRCKr3e0n","colab_type":"text"},"cell_type":"markdown","source":"##3.Convolutional Neural Network Model"},{"metadata":{"id":"kpkANohy3TaJ","colab_type":"code","outputId":"956ee2f4-8139-4189-8f92-b2382119e70d","colab":{"base_uri":"https://localhost:8080/","height":88},"trusted":true},"cell_type":"code","source":"#CNN Architecture is In -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*2 -> \n                           #Flatten -> Dense -> Dropout -> Out\nmodel = tf.keras.Sequential()\n\nmodel.add(layers.Conv2D(filters=32, kernel_size=(5,5), padding='Same', \n                       activation=tf.nn.relu, input_shape = (28,28,1)))\nmodel.add(layers.Conv2D(filters=32, kernel_size=(5,5), padding='Same', \n                       activation=tf.nn.relu))\nmodel.add(layers.MaxPool2D(pool_size=(2,2)))\nmodel.add(layers.Dropout(0.25))\n\n\nmodel.add(layers.Conv2D(filters=64, kernel_size=(3,3), padding='Same', \n                       activation=tf.nn.relu, input_shape = (28,28,1)))\nmodel.add(layers.Conv2D(filters=64, kernel_size=(3,3), padding='Same', \n                       activation=tf.nn.relu))\nmodel.add(layers.MaxPool2D(pool_size=(2,2),strides=(2,2)))\nmodel.add(layers.Dropout(0.25))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(256,activation=tf.nn.relu))\nmodel.add(layers.Dropout(0.25))\nmodel.add(layers.Dense(10,activation=tf.nn.softmax))","execution_count":null,"outputs":[]},{"metadata":{"id":"VuN1-e0uPCWy","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#Defining Optimizer\n\noptimizer = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"wiwSpyiYPcIb","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#Compiling Model\n\nmodel.compile(optimizer = optimizer, loss='categorical_crossentropy', \n             metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"id":"FqExgW5DQoab","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#Setting Learning rate annealer\n\nlearning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc',\n                                           patience=3,\n                                           verbose=1,\n                                           factor=0.5,\n                                           min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"id":"VpoFy1wQRJf8","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"epochs=20\nbatch_size = 86","execution_count":null,"outputs":[]},{"metadata":{"id":"E573js59RQVn","colab_type":"text"},"cell_type":"markdown","source":"###Data Augmentation"},{"metadata":{"id":"lTi2iTMKROwt","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"id":"qkX93W-P4iYl","colab_type":"text"},"cell_type":"markdown","source":"###Model Fitting"},{"metadata":{"id":"BYvD4vpOz3_L","colab_type":"code","outputId":"02ebfa24-d959-4b82-8aa5-f83f18c98a0a","colab":{"base_uri":"https://localhost:8080/","height":799},"trusted":true},"cell_type":"code","source":"# Fit the model\nhistory = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"id":"PMKZhU75z-b6","colab_type":"code","outputId":"4801351f-7ead-469c-a528-f418abc68eb6","colab":{"base_uri":"https://localhost:8080/"},"trusted":true},"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"oTsAl3Ur3n9d","colab_type":"text"},"cell_type":"markdown","source":"###Confusion Matrix Plotting\nIt is used for grphical representation of performance of the model. It shows the performance of Model in predicting every class."},{"metadata":{"_execution_state":"idle","_uuid":"16e161179bf1b51ba66c39b2cead883f1db3a9c7","_cell_guid":"11361e73-8250-4bf5-a353-b0f8ea83e659","trusted":true,"id":"IK6-TyagG4G4","colab_type":"code","outputId":"29d7f070-8773-4f39-d2fe-6f02d0d2bfa1","colab":{"base_uri":"https://localhost:8080/"}},"cell_type":"code","source":"# Look at confusion matrix \n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(Y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) ","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"e7a3d6449b499a29db224e42e950f21ca1ec4e36","_cell_guid":"7b0f31b8-c18b-4529-b0d8-eb4c31e30bbf","trusted":true,"id":"tV4wnLTQG4G8","colab_type":"code","outputId":"718bdde1-4a6b-45f9-c5e9-18e93cb845cb","colab":{"base_uri":"https://localhost:8080/"}},"cell_type":"code","source":"# Display some error results \n\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted :{} True :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors \nmost_important_errors = sorted_dela_errors[-6:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"7f17e7bf0a54a01a52fef2d554780f6bc6580dc6","_cell_guid":"05ff3b9f-c3bb-4cec-a8c2-2c128e8f15b3","trusted":true,"id":"N9KswR97G4G-","colab_type":"code","colab":{}},"cell_type":"code","source":"# predict results\nresults = model.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"369dfaab09240f3f12bcff91953ffd315ab84985","_cell_guid":"b5f1f39f-13b8-439a-8913-0f120e3d47a9","trusted":true,"id":"qvS8FrQEG4HC","colab_type":"code","colab":{}},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Copy of Digit Recognizer.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}