{"metadata": {"language_info": {"mimetype": "text/x-python", "pygments_lexer": "ipython3", "codemirror_mode": {"name": "ipython", "version": 3}, "nbconvert_exporter": "python", "name": "python", "file_extension": ".py", "version": "3.6.1"}, "kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}}, "nbformat": 4, "nbformat_minor": 0, "cells": [{"metadata": {"collapsed": false, "_cell_guid": "911650d6-8daf-44de-bf53-d5260ced4cd4", "_uuid": "8cba1c2ae7222531442de2f0cf9f93606e8206cc", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "This notebook entry demonstrates how to use Kera's activation maps to analyze why a digit can be mislabelled as another number.  The activation or feature maps are the output of convolution and pooling layers in a network from a given input.", "cell_type": "markdown"}, {"metadata": {"collapsed": false, "_cell_guid": "2af7773c-c1a0-4729-a1f9-120e60a8d686", "_uuid": "74408a483d3be0aeb08312e5f0748e4a08d32988", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "import itertools\nimport pandas as pd\nimport numpy as np\nimport math\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nfrom keras import layers\nfrom keras import models\nfrom keras.utils import to_categorical\nfrom keras import optimizers\n\nimport seaborn as sns\nsns.set(style='white', context='notebook', palette='deep')\nnp.random.seed(42)", "cell_type": "code"}, {"metadata": {"_cell_guid": "4c4259c9-29cd-4e4a-9395-d60eb22308f4", "_uuid": "65b5470c6341c63e19cc75ff81d2fbd19b5be198", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "train_dir = '../input/train.csv'\ntrain_df = pd.read_csv(train_dir)\ntest_dir = '../input/test.csv'\ntest_df = pd.read_csv(test_dir)\n\ntrain_labels = train_df['label']\ntrain_images = train_df.drop(labels=['label'], axis=1)\ndel train_df\n\ntrain_images = train_images / 255.0\ntest_df = test_df / 255.0\n\ntrain_images = train_images.values.reshape(-1, 28, 28,1)\ntest_images = test_df.values.reshape(-1, 28, 28, 1)\ndel test_df\n\ntrain_labels = to_categorical(train_labels, num_classes=10)\n\nrandom_seed = 42\n# Split the training set to one training and validation sets for the fitting.\ntrain_images, validation_images, train_labels, validation_labels = train_test_split(\n    train_images, train_labels, test_size=0.1, random_state=random_seed)", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "db7cfdc3-d8d6-4879-86ab-5b859f258f6e", "_uuid": "a1b801b5db9680d5bf3dd00729e75a9054e63307", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Create the model\nmodel = models.Sequential()\nmodel.add(layers.Conv2D(32, (3,3), activation='relu', input_shape=(28,28,1)))\nmodel.add(layers.MaxPooling2D(2,2))\nmodel.add(layers.Conv2D(64, (3,3), activation='relu'))\nmodel.add(layers.MaxPooling2D(2,2))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\n\nmodel.summary()", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "d7c6cce7-489d-49e5-a068-58810441a364", "_uuid": "f7349c748bf40a4f6f1c5a13fcc113b7cb129023", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Default RMSProp parameters\nfrom keras import optimizers\noptimizers = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\nmodel.compile(optimizer=optimizers,\n             loss='categorical_crossentropy',\n             metrics=['accuracy'])", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "f1732f4f-b556-4614-a07b-11faaa0cbb8a", "_uuid": "32c6531be92669e03a291c9c302217680ab6a154", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "epochs = 10\nbatch_size=64\n\n# Trains the model for a fixed number of epochs.\nhistory = model.fit(train_images, \n          train_labels, \n          epochs=epochs, \n          batch_size=batch_size, \n          verbose=0,\n          validation_data=(validation_images, validation_labels))", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "83680bb2-2a59-4009-9067-8b88bde60208", "_uuid": "cd4615500ea3dbfee0919a39f861cfdedfccf60e", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# The graph seems to suggest a underfitting. Add more layers so that the network could learn more. \nacc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nfig, ax = plt.subplots(2, 1)\nax[0].plot(loss, color='b', label='Training loss')\nax[0].plot(val_loss, color='r', \n           label='Validation loss', axes=ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(acc, color='b', label=\"Training accuracy\")\nax[1].plot(val_acc, color='r', label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "0dab8feb-e868-4cde-841e-f2ac28df26dc", "_uuid": "dd3a5a97f603b2a35e98398705a3ae8de80286cf", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "### Analysis\nNotes:   \n1) The code for finding and displaying important errors are copied from the good work of Yassine Ghozam's 'Introduction to CNN Keras\" kernel.  \n2) The function \"plot_confusion_matrix\" is taken from the example, http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html  \n3) The displaying of activation layers are from the wonderful book \"Deep Learning with Python\" from Manning Publishing.  ", "cell_type": "markdown"}, {"metadata": {"collapsed": false, "_cell_guid": "cbde44e0-688a-447e-90ff-5e14193c8afc", "_uuid": "d723888ea4d5ef3c3eb63da8fbf2e5213e0824ec", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Look at confusion matrix\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    #print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "7b13b8de-0533-45ba-91de-99ab729ce6eb", "_uuid": "9f1126e79092f11a29cbf60755b89667ce7f4ef7", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Predict the values (in propabilities) from the validation dataset\nval_img_pred = model.predict(validation_images)\n\n# Convert one-hot-vectors to predictions labels\n# Map the prediction labels from the column with the highest probability (Each column represents one label)\nval_img_pred_labels = np.argmax(val_img_pred, axis=1)\n\n# Convert the one-hot-vectors to validation observations (mapping the true labels)\nval_img_true = np.argmax(validation_labels, axis=1)\n\n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(val_img_true, val_img_pred_labels)\n\n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes=range(10))\n\n# The matrix shows several instances where predicted number is 5 but mislabeled as 3 or the other way around. \n# Therefore, we can use those instances in our analysis below.", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "087e5dab-adb6-4a99-a119-9ecde3758499", "_uuid": "5637983477bd2132b875a58dda7ecc82b55960a3", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "def display_errors(errors_index, img_errors, pred_errors, obs_errors, nrows=2, ncols=3):\n    \"\"\"\n    This function shows 6 images with their predicted and real labels\n    \"\"\"\n    n = 0\n    \n    fig, ax = plt.subplots(nrows, ncols, sharex=True, sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label : {}\\nTrue label : {}\".format(\n                pred_errors[error], obs_errors[error]))\n            n += 1", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "b615e097-491d-4cf3-8bd7-659ecebb9c08", "_uuid": "b25013a60a0be0f1ff9fb8a8eca7f78e5f407e46", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "### Find most important errors", "cell_type": "markdown"}, {"metadata": {"collapsed": false, "_cell_guid": "cd5ba611-9799-4dd3-a69d-a0b3cc8775ad", "_uuid": "3f963b391fd6cd4e311ad422648e8ff0a8c79739", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Generate the error masks by comparing the predicted labels against true labels\nerrors = (val_img_pred_labels - val_img_true != 0)\n\n# Predicted labels that only include errors\nval_img_pred_labels_errors = val_img_pred_labels[errors]\n\n# Predicted probabilities that only include errors still in one-hot-vector form\nval_img_pred_errors = val_img_pred[errors]\n\n# True Validation Labels that were wrongly predicted\nval_img_true_errors = val_img_true[errors]\n\n# Validation data with wrong predictions\nvalidation_images_errors = validation_images[errors]\n\n# Probabilities of the wrong predicted numbers\nval_img_pred_errors_prob = np.max(val_img_pred_errors, axis=1)\n\n# Predicted probabilities of the true values in the \"val_img_pred_errors\" error set\n# ie. Predicted label 3 and true label 5\n#    0    1    2      3      4     5   ...\n# 0.01 0.01 0.01   0.89   0.01  0.01\n#                (pred)        (true)\ntrue_prob_errors = np.diagonal(np.take(val_img_pred_errors, \n                                       val_img_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = val_img_pred_errors_prob - true_prob_errors\n\n# Sorted indexes (in list data type) of the delta prob errors, starting from the least difference\nmost_important_errors = np.argsort(delta_pred_true_errors)\n\n# Read the top 25 errors starting from last index in the array\n# most_important_errors = sorted_delta_errors[-25:]\ncol_row_cnt = math.floor(math.sqrt(most_important_errors.shape[0]))\n\n## Show the top 6 errors\ndisplay_errors(most_important_errors, \n               validation_images_errors, \n               val_img_pred_labels_errors, \n               val_img_true_errors,\n               nrows=col_row_cnt,\n               ncols=col_row_cnt)", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "d6fea177-201c-4b51-a64a-00fbfc9b9c50", "_uuid": "d07f445e090689baa469b7a9dcc9f174a6af7c90", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Find the indexes from the error mask\n# errors_idxs - contains all the indexes of true validation labels that have wrong predictions.\nerrors_idxs =  np.array(errors.nonzero())[0,:]", "cell_type": "code"}, {"metadata": {"collapsed": false, "_uuid": "169ee722fcb6deec1c8ce5ddcc4e9902bf8b568a", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "### Find an index of a wrongly predicted number 3 that was labelled as 5 or predicted as 5 but should be 3.", "cell_type": "markdown"}, {"metadata": {"collapsed": false, "_uuid": "e00b9cfa8930d001aba15b21191e4c6dc612601a", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "mst_impt_error_idx = [mst_impt_error_idx for mst_impt_error_idx in most_important_errors \n                        if (\n                            (val_img_pred_labels_errors[mst_impt_error_idx] == 5 \n                               and val_img_true_errors[mst_impt_error_idx]==3) or\n                            (val_img_pred_labels_errors[mst_impt_error_idx] == 3 \n                             and val_img_true_errors[mst_impt_error_idx]==5)\n                        )\n                     ][0]\n\nprint(\"Predicted number: \", val_img_pred_labels_errors[mst_impt_error_idx])\nprint(\"True label: \", val_img_true_errors[mst_impt_error_idx])", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "9240bc1e-1c71-46a6-b2ea-742545c0fc2e", "_uuid": "fca2647062123e0dff16f662ebd2b725050d3c41", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Show the target error image\nplt.imshow((validation_images_errors[mst_impt_error_idx]).reshape((28,28)))\n\n# Map the mst_impt_error_idx to the index of our official validation set (validation_images). \n# Note that the value of mst_impt_error_idx is comes from our error set (validation_images_errors). \nerror_idx_to_analyze = errors_idxs[mst_impt_error_idx]", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "daf84054-dc20-425f-9c9e-95ebb4dfd592", "_uuid": "075c9902a53d98ea9588454fcbe516f5ffbae2e1", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Lets look at a sample activation layer of the above error \n# Extracts the outputs of the top 4 layers:\nlayer_outputs = [layer.output for layer in model.layers[:4]]\n\n# Creates a model that will return these outputs, given the model input:\nactivation_model = models.Model(input=model.input, output=layer_outputs)\n\n# This will return a list of 4 Numpy array:\n# one array per layer activation\nactivations = activation_model.predict(validation_images)\nfirst_layer_activation = activations[0]\n\nplt.matshow(first_layer_activation[error_idx_to_analyze,:,:,4], cmap='viridis')", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "4e21c63d-d332-4d7c-bc99-be773f879f42", "_uuid": "efd8b4f94cce4b3caf735f033430c2f3179c66ca", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "def display_activation_layers(numberIdx):\n    # These are the names of the layers, so can have them as part of our plot\n    layer_names = []\n    for layer in model.layers[:4]:\n        if isinstance(layer, layers.Conv2D):\n            layer_names.append(layer.name)\n\n    images_per_row = 16\n\n    #numberIdx = 2492\n    # Now let's display our feature maps\n    for layer_name, layer_activation in zip(layer_names, activations):\n        # this is the number of features in the feature map\n        n_features = layer_activation.shape[-1]\n        # the feature map has shape (1, size, size, n_features)\n        size = layer_activation.shape[1]\n        # we will tile the activation channels in this matrix\n        n_cols = int(n_features / images_per_row)\n\n        display_grid = np.zeros((size * n_cols, images_per_row * size))\n        # we'll tile each filter into this big horizontal grid\n        for col in range(n_cols):\n            for row in range(images_per_row):\n                channel_image = layer_activation[numberIdx, :, :, col * images_per_row + row]\n                # Post-process the feature to make it visually palatable\n                channel_image -= channel_image.mean()\n                channel_image /= channel_image.std()\n                channel_image *= 64\n                channel_image += 128\n                channel_image = np.clip(channel_image, 0, 255).astype('uint8')\n                display_grid[col * size : (col + 1) * size, row * size : (row + 1) * size] = channel_image\n        # display the grid\n        scale = 1. / size\n        plt.figure(figsize=(scale * display_grid.shape[1], scale * display_grid.shape[0]))\n        plt.title(layer_name)\n        plt.grid(False)\n        plt.imshow(display_grid, aspect='auto', cmap='viridis')", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "9ab5d9a9-3737-451c-a799-f5d811875fe3", "_uuid": "b6299a37c2024eab1398dfb354684e82cabd54f2", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Activation map of the wrongly predicted label\ndisplay_activation_layers(error_idx_to_analyze) ", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "4e0b9041-a325-47b6-ba99-4fd9a56f7ae9", "_uuid": "208891f25bcec9a1dfcf2b0bf0788fa5069df107", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Now let's look at the activation map of a validation label 5 that was correctly predicted\n# First, find all the indexes of true label 5\nmsk1 = (val_img_true == 5)\nidxs1 = np.array(msk1.nonzero())[0,:]\nidx1_to_analyze = [idx for idx in idxs1 if np.any(errors_idxs != idx)][0]\nplt.imshow((validation_images[idx1_to_analyze]).reshape((28,28)))\n\n# Activation layers mapping for a correct prediciton of validation label 5\ndisplay_activation_layers(idx1_to_analyze)", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "d0d086cf-cb20-4e4b-84cd-a5396e8512bd", "_uuid": "b3e5956ead49c45658a034518e8ffb197819e65d", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "# Now let's look at the activation map of a validation label 3 that was correctly predicted\n# Lets find the indexes of all true label 3\nmsk2 = (val_img_true == 3)\nidxs2 = np.array(msk2.nonzero())[0,:]\n\n# Find the first label 3 index that does not appear in the error set\nidx2_to_analyze = [idx for idx in idxs2 if np.any(errors_idxs != idx)][0]\nplt.imshow((validation_images[idx2_to_analyze]).reshape((28,28)))\n\n# Activation layers mapping for a correct prediciton of validation label 3 in array index 12\ndisplay_activation_layers(idx2_to_analyze)", "cell_type": "code"}, {"metadata": {"collapsed": false, "_cell_guid": "b3862f4f-d176-4327-9ec1-a3529830f6a0", "_uuid": "ec2543f2d14402c979a2f0da00bfa774acc7aa0f", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "### Conclusion  \nThe activation maps helps us infer why number 3 is mistaken as 5 or 5 wrongly labelled as 3. This maybe due to the similarities of their second activation layers.  The digit 5 activation images are shaped like letter \"S\" while the digit 3 usually shaped as 2 vertically stacked inverted small letter \"c\".", "cell_type": "markdown"}, {"metadata": {"collapsed": false, "_cell_guid": "3f210b4d-1601-4987-a681-3ac27fa54067", "_uuid": "da1b70ae32a67122b24e00d5960566a5c4e7962e", "trusted": false, "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "", "cell_type": "code"}, {"metadata": {"collapsed": false, "_uuid": "183c8089bef77bb5cad0eea202d3af7c6c233040", "_execution_state": "idle"}, "execution_count": null, "outputs": [], "source": "", "cell_type": "code"}]}