{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nfrom keras.utils.np_utils import to_categorical # Label encoding\n\nfrom sklearn.model_selection import train_test_split # Split Data\n\n# Modelling\nfrom tensorflow import keras\nfrom keras import layers\n\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\n\nfrom keras import models\n\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"X_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nX_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\nprint('Shape of the training data: ', X_train.shape)\nprint('Shape of the test data: ', X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### data distribution"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"f = plt.figure(figsize=(10,6))\ncounts = X_train['label'].value_counts().sort_index()\nsns.barplot(counts.index, counts.values)\n\nfor i in counts.index:\n    plt.text(i,counts.values[i]+50,str(counts.values[i]),horizontalalignment='center',fontsize=14)\n\n\nplt.tick_params(labelsize = 14)\nplt.xticks(counts.index)\nplt.xlabel(\"Digits\",fontsize=16)\nplt.ylabel(\"Frequency\",fontsize=16)\nplt.title(\"Frequency Graph training set\",fontsize=20)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"y_train = X_train['label']\nX_train.drop(labels = ['label'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"### Nulls=? & Normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Null values in training data: ',X_train.isna().any().sum())\nprint('Null values in test data: ',X_test.isna().any().sum())\n\nX_train = X_train / 255.0\nX_test = X_test / 255.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Re-Shape data"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.values.reshape(-1, 28, 28, 1)\nX_test = X_test.values.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Label Encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = to_categorical(y_train, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling data\n<img src=\"https://i.imgur.com/PFn0LrM.jpg\">"},{"metadata":{},"cell_type":"markdown","source":"### Train-Validation Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### keras-NN Base"},{"metadata":{"_kg_hide-output":false,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"model = keras.models.Sequential([layers.Conv2D(32, (3,3), activation=\"relu\",padding='same', input_shape=(28,28,1)),\n                                 layers.MaxPooling2D(2,2),\n                                 layers.Dropout(0.25),\n                                  layers.Conv2D(64, (3,3), activation=\"relu\",padding='same'),\n                                 layers.MaxPooling2D(2,2),\n                                 layers.Dropout(0.25),\n                                 layers.Flatten(),\n                                  layers.BatchNormalization(),\n                                 layers.Dense(128, activation=\"elu\", kernel_initializer=\"he_normal\"),\n                                  layers.BatchNormalization(),\n                                 layers.Dense(32, activation=\"elu\", kernel_initializer=\"he_normal\"),\n                                  layers.BatchNormalization(),\n                                 layers.Dense(10, activation=\"softmax\")])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### keras optimizer, loss-func, metrics & callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = Adam(learning_rate=0.001, epsilon=1e-07)\n\nmodel.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics=['accuracy'])\n\nearlyStopping = EarlyStopping(monitor='val_accuracy', patience=10, verbose=0, mode='auto')\nmcp = ModelCheckpoint('.mdl_wts.hdf5', save_best_only=True, monitor='val_accuracy', mode='auto')\nreduce_lr_loss = ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='auto')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image Data Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n    rotation_range=5,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    shear_range=5,\n    zoom_range=0.1)\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit model"},{"metadata":{},"cell_type":"markdown","source":"as we are using early stopping, model will converge early, so we can go for larger epoches"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"Batch_size=100\nEpochs = 100\nhistory = model.fit_generator(datagen.flow(X_train, y_train, batch_size=Batch_size),\n                              epochs = Epochs, \n                              validation_data = (X_val,y_val),\n                              verbose = 2, \n                              steps_per_epoch=X_train.shape[0]//Batch_size, \n                             callbacks = [earlyStopping, mcp, reduce_lr_loss])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Performance at glance"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"f = plt.figure(figsize=(20,7))\n\nf.add_subplot(121)\n# plt.figure(figsize=(6, 4))\nplt.plot(history.history['accuracy'], color='b', label=\"Train\")\nplt.plot(history.history['val_accuracy'], color='r',label=\"Validation\")\nplt.legend(loc='best',fontsize=18)\nplt.title('Accuracy Curve',fontsize=25)\n\n\n\nf.add_subplot(122)\n# plt.figure(figsize=(6, 4))\nplt.plot(history.history['loss'], color='b', label=\"Train\")\nplt.plot(history.history['val_loss'], color='r', label=\"validation\")\nplt.legend(loc='best',fontsize=18)\nplt.title('Loss Curve',fontsize=25)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Confusion Matrix"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(10,8.5))\ny_pred = model.predict(X_val)\ny_pred = np.argmax(y_pred, axis=1)\ny_val_test = np.argmax(y_val, axis=1)\ncm = confusion_matrix(y_val_test, y_pred)\nsns.heatmap(cm, annot=True, fmt='g', cbar=False)\nplt.xlabel('Predicted')\nplt.ylabel('True')\nplt.title('Confusion Matrix for predicted and true labels')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Choosing the best model weights"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"model.load_weights(filepath = '.mdl_wts.hdf5')\n\nscores = model.evaluate(X_val, y_val, callbacks = [earlyStopping, mcp, reduce_lr_loss])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Number of wrong predictions in validation for each digit"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"cnt_error = []\nfor (a, b) in zip(y_val_test, y_pred):\n    if a == b: continue\n    cnt_error.append( a )\n\n    \ncnt_error = np.unique(cnt_error, return_counts = True)\ncnt_error = pd.Series(cnt_error[1],index=cnt_error[0])\nf = plt.figure(figsize=(10,6))\nsns.barplot(cnt_error.index, cnt_error.values)\n\n# for i in counts.index:\n#     plt.text(i,cnt_error.values[i]+0.2,str(cnt_error.values[i]),horizontalalignment='center',fontsize=14)\n\n\nplt.tick_params(labelsize = 14)\nplt.xticks(cnt_error.index)\nplt.xlabel(\"Digits\",fontsize=16)\nplt.ylabel(\"Frequency\",fontsize=16)\nplt.title(\"Worng Predictions\",fontsize=20)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization"},{"metadata":{},"cell_type":"markdown","source":"#### Validation set images"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"rows = 3\ncols = 4\n\nplt.figure(figsize=(10,7))\nfor i in range(rows*cols):\n    ax = plt.subplot(rows, cols, i+1)\n    plt.axis('off')\n    ax.set_xticks([])\n    ax.set_yticks([])\n    plt.imshow(X_val[i].reshape(28, 28), cmap='Blues')\n    plt.title('Predicted: {}   True: {}'.format(y_pred[i], y_val_test[i])\n              ,y=-0.15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Wrong Predictions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"ls = np.array(y_pred - y_val_test) # Subtract true values from predicted --> All non-zero results were incorrectly predicted\n\nnonzero_pred = np.nonzero(ls)[0]\n\nrows = 3\ncols = 4\n\nplt.figure(figsize=(10,7))\nfor i in range(rows*cols):\n    ax = plt.subplot(rows, cols, i+1)\n    plt.axis('off')\n    ax.set_xticks([])\n    ax.set_yticks([])\n    plt.imshow(X_val[nonzero_pred[i]].reshape(28, 28), cmap='Reds')\n    plt.title('Predicted: {}  True: {}'.format(y_pred[nonzero_pred[i]], y_val_test[nonzero_pred[i]]), y=-0.15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Test Set predictions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test = X_test[0:30,:,:,:]\ntest_pred = np.argmax(model.predict(X_test), axis=1)\nrows = 3\ncols = 4\nplt.figure(figsize=(10,7))\nfor i in range(rows*cols):\n    ax = plt.subplot(rows,cols, i+1)\n    plt.axis('off')\n    ax.set_xticks([])\n    ax.set_yticks([])\n    plt.imshow(X_test[i].reshape(28, 28), cmap='bone')\n    plt.title('Predicted: {}'.format(test_pred[i]),y=-0.15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FILTERS\n\n* This neural net uses two conv2d layers each of size 32 and 64 respectively\n\n* These filters are learnable filters that learn some visual feature as the model gets trained over images across epochs."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for layer in model.layers:\n    if'conv' in layer.name:\n        filters, biases = layer.get_weights()\n        print('Layer: ', layer.name, filters.shape)\n#         f_min, f_max = filters.min(), filters.max()\n#         filters = (filters - f_min) / (f_max - f_min)\n        \n        print('Filter size: (', filters.shape[0], ',', filters.shape[1], ')')\n        print('Channels in this layer: ', filters.shape[2])\n        print('Number of filters: ', filters.shape[3])\n        \n        count = 1\n        plt.figure(figsize = (18, 4))\n        \n        for i in range(filters.shape[3]):\n            ax= plt.subplot(4, filters.shape[3]/4, count)\n            ax.set_xticks([])\n            ax.set_yticks([])\n            plt.imshow(filters[:,:,0, i], cmap=plt.cm.binary)\n            count+=1\n            \n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Output of these filters for each layer"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"print('total number of layers',len(model.layers))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"lets consider all the layers above the flatten layer\n\nlets use keras **Model** class to get the output state of each layer for an input"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract outputs of top 6 layers\nlayer_outputs = [layer.output for layer in model.layers[0:6]]\n\n# Create a model that will return these outputs given the model input\nactivation_model = models.Model(inputs = model.input, outputs = layer_outputs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"consider a sample from test set"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.imshow(X_test[4].reshape(28, 28), cmap = plt.cm.binary)\nplt.xticks([])\nplt.yticks([])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"img_tensor = X_test[4].reshape(-1, 28, 28, 1)\nactivations = activation_model.predict(img_tensor)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"layers shapes in our NN from top"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for i in range(len(activations)):\n    print(activations[i].shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"lets observe the outputs of 1st and 16th filters"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"first_layer_activation = activations[0]\n\nax = plt.subplot(1, 2, 1)\nax.set_xticks([])\nax.set_yticks([])\nplt.imshow(first_layer_activation[0,:,:,0], cmap = plt.cm.binary)\nplt.title('1st Filter o/p')\n\nax = plt.subplot(1, 2, 2)\nax.set_xticks([])\nax.set_yticks([])\nplt.imshow(first_layer_activation[0,:,:,16], cmap = plt.cm.binary)\nplt.title('16th Filter o/p')\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## All Activation Layers"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"layer_names = []\nfor layer in model.layers[:6]:\n    layer_names.append(layer.name)\n\nfor activation_layer, layer_name in zip(activations, layer_names):\n\n    n_features = activation_layer.shape[3]\n    feat_per_row = 16\n    rows = n_features//feat_per_row\n    size = activation_layer.shape[1]\n    \n    print(layer_name)\n    plt.figure(figsize=(20, rows))\n    for i in range(n_features):\n        ax = plt.subplot(rows, feat_per_row, i+1)\n        ax.set_xticks([])\n        ax.set_yticks([])\n        plt.imshow(activation_layer[:,:,:,i].reshape(size, size), cmap = plt.cm.binary)\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions = pd.DataFrame({\"ImageId\": list(range(1,len(test_pred)+1)),\n    \"Label\": test_pred})\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}