{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Welcome to digital recognizer with using CNN"},{"metadata":{},"cell_type":"markdown","source":"**I would really appreciate your votes if you liked this notebook!**"},{"metadata":{},"cell_type":"markdown","source":"*First we need import all required libraries*"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport h5py\nimport itertools\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization\nfrom tensorflow.keras.losses import CategoricalCrossentropy as CC\nfrom tensorflow.keras.losses import SparseCategoricalCrossentropy as SCC\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils.np_utils import to_categorical\n\n%matplotlib inline\nnp.random.seed(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load the data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Processing the data"},{"metadata":{},"cell_type":"markdown","source":"## Data separation"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train.iloc[:,1:]\nY_train = train.iloc[:,0]\n\nprint(X_train.shape, ', ', Y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train/255\ntest = test/255\n\nprint(X_train.shape, ', ', test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data reshaping"},{"metadata":{},"cell_type":"markdown","source":"*To process the data we need to reshape 784 pixels of each entry into (28, 28, 1)*"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)\n\nprint(X_train.shape,', ', test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data splitting"},{"metadata":{"trusted":true},"cell_type":"code","source":"random_seed = 2\nY_train = to_categorical(Y_train, num_classes = 10)\n\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, \n                                                  random_state = random_seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*Here we look at the examples of each digit*"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(3,3,figsize=(10,10))\nfig.suptitle('Example of each number', size = 20)\nnums, x = range(10), 0\nfor i in range(3):\n    for j in range(3):\n        label = nums[x]\n        numbers_pixels = train[train['label'] == label]\n        img = numbers_pixels.iloc[0,1:].values\n        img = img.reshape((28,28))\n        ax[i][j].imshow(img, cmap = plt.cm.binary)\n        ax[i][j].set_title(\"Number: \" + str(label))\n        x += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initial model"},{"metadata":{},"cell_type":"markdown","source":"*This is a model with 11 layers such as convolutional layer, pooling layer, flatten layer and fully-connected(dense) layer. Final layer will make a predictions*"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5, 5), padding = \"Same\", activation = \"relu\", \n                 input_shape = (28, 28, 1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5, 5), padding = \"Same\", activation = \"relu\"))\nmodel.add(MaxPooling2D(pool_size = (2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3, 3), padding = \"Same\", activation = \"relu\"))\nmodel.add(Conv2D(filters = 64, kernel_size = (3, 3), padding = \"Same\", activation = \"relu\"))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation=\"relu\"))\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(10, activation=\"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compile the model"},{"metadata":{},"cell_type":"markdown","source":"*We can use different optimizers such as Adam, SGD, RMSprop, Adadelta, Adagrad, Adamax, Nadam, Ftrl. In this model we use RMSprop optimizer*\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = RMSprop(lr = 0.001, rho = 0.9, epsilon = 1e-08, decay = 0.0)\nmodel.compile(optimizer = opt, loss = SCC(), metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*Using the method ```summary()``` we can see layers' types, output shape and amount of parameters*"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set a learning rate annealer"},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor = \"val_acc\", patience = 3, \n                                            verbose = 1, factor = 0.5, min_lr = 0.00001)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator = ImageDataGenerator(\n    featurewise_center=False,  # set input mean to 0 over the dataset\n    samplewise_center=False,  # set each sample mean to 0\n    featurewise_std_normalization=False,  # divide inputs by std of the dataset\n    samplewise_std_normalization=False,  # divide each input by its std\n    zca_whitening=False,  # apply ZCA whitening\n    rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n    zoom_range=0.1,  # Randomly zoom image\n    width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip=False,  # randomly flip images\n    vertical_flip=False,\n)  # randomly flip images\n\ndata_generator.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\nepochs = 30\nhistory = model.fit_generator(\n    data_generator.flow(X_train, Y_train, batch_size = batch_size),\n    epochs = epochs,\n    validation_data = (X_val, Y_val),\n    verbose = 2,\n    steps_per_epoch = X_train.shape[0] // batch_size,\n    callbacks = [learning_rate_reduction],\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluate the model"},{"metadata":{},"cell_type":"markdown","source":"## Plot the loss and accuracy curves for training and validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2, 1)\nax[0].plot(history.history[\"loss\"], color=\"b\", label=\"Training loss\")\nax[0].plot(history.history[\"val_loss\"], color=\"g\", label=\"validation loss\", axes=ax[0])\nax[0].legend(loc=\"best\", shadow=True)\n\nax[1].plot(history.history[\"accuracy\"], color=\"b\", label=\"Training accuracy\")\nax[1].plot(history.history[\"val_accuracy\"], color=\"g\", label=\"Validation accuracy\")\nax[1].legend(loc=\"best\", shadow=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Confusion matrix"},{"metadata":{},"cell_type":"markdown","source":"*The function `plot_confusion_matrix()` prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`*"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes, normalize = False, title = \"Confusion matrix\", cmap = plt.cm.Blues):\n    plt.imshow(cm, interpolation=\"nearest\", cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=0)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.0\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(\n            j,\n            i,\n            cm[i, j],\n            horizontalalignment=\"center\",\n            color=\"white\" if cm[i, j] > thresh else \"black\",\n        )\n\n    plt.tight_layout()\n    plt.ylabel(\"True label\")\n    plt.xlabel(\"Predicted label\")\n    \n    \n# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors\nY_pred_classes = np.argmax(Y_pred, axis=1)\n# Convert validation observations to one hot vectors\nY_true = np.argmax(Y_val, axis=1)\n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes)\n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes=range(10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display some error results"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# Errors are difference between predicted labels and true labels\nerrors = Y_pred_classes - Y_true != 0\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]\n\n\ndef display_errors(errors_index, img_errors, pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows, ncols, sharex = True, sharey = True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row, col].imshow((img_errors[error]).reshape((28, 28)))\n            ax[row, col].set_title(\n                \"Predicted label :{}\\nTrue label :{}\".format(\n                    pred_errors[error], obs_errors[error]\n                )\n            )\n            n += 1\n\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors, axis=1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors\nmost_important_errors = sorted_dela_errors[-6:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predictions and accuracy"},{"metadata":{},"cell_type":"markdown","source":"*Generate predictions on the test data using `predict()`*"},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities = model.predict(test)\ny_pred = np.argmax(probabilities, axis=1)\nprint(y_pred.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*We evaluate the model on the train data via evaluate()*"},{"metadata":{},"cell_type":"markdown","source":"# Initial submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'ImageId': np.arange(1, 28001), 'Label': y_pred})\nsubmission.to_csv(\"submission2.csv\", index = False)\nprint(\"Your submission was successfully saved!\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}