{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction to Computer Vision: MNIST Challenge\nHi!\n\nThis is my modified original MNIST solution \n\n\n\n__________________________\n\n\n\n\n### Table of interest:\n> ### 1. Starting with Kaggle - importing data \n> ### 2. Visualize some examples from the dataset.\n> ### 3. Defining the architecture\n> ### 4. Train the model using data augmentation\n> ### 5. After train visualizations\n> > #### 5.1 Loss graph visualizations\n> > #### 5.2 Prediction images visualization\n> > #### 5.3 Confusion matrix\n> > #### 5.4 Miss-labeled data visualization\n> ### 6. Submission\n\n"},{"metadata":{},"cell_type":"markdown","source":"## 1. Starting with Kaggle - importing data \n\nImportant: Switch settings to GPU"},{"metadata":{},"cell_type":"markdown","source":"### Importing libraries"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\n\n# Input data files are available in the \"../input/\" directory.\nimport os\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.\n\n# This is a bit of magic to make matplotlib figures appear inline in the notebook\n# rather than in a new window.\n%matplotlib inline\nplt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\nplt.rcParams['image.interpolation'] = 'nearest'\nplt.rcParams['image.cmap'] = 'gray'\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Importing the data the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.utils.np_utils import to_categorical\nfrom sklearn.model_selection import train_test_split\n\n# Import Data\ntrain = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\ntest= pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nprint(\"Train size:{}\\nTest size:{}\".format(train.shape, test.shape))\n\n# Transform Train and Test into images\\labels.\nx_train = train.drop(['label'], axis=1).values.astype('float32') # all pixel values\ny_train = train['label'].values.astype('int32') # only labels i.e targets digits\nx_test = test.drop(['id'], axis=1).values.astype('float32') # all pixel values\n\nx_train = x_train.reshape(x_train.shape[0], 28, 28) / 255.0\nx_test = x_test.reshape(x_test.shape[0], 28, 28) / 255.0\n\nx_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.15, random_state=42)\n\nprint(x_train.shape)\nprint(x_val.shape)\nprint(y_train.shape)\nprint(x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Visualize some examples from the dataset.\nShowing some example per class\n<br>\nNotcise that 7 and 3 are looking alike"},{"metadata":{"trusted":true},"cell_type":"code","source":"# classes for title\n# num classes for amount of examples\nclasses = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\nprint(x_train.shape)\nnum_classes = len(classes)\nsamples_per_class = 7\nplt.figure(0)\nfor y, cls in enumerate(classes):\n    idxs = np.flatnonzero(y_train == y)\n    idxs = np.random.choice(idxs, samples_per_class, replace=False)\n    for i, idx in enumerate(idxs):\n        plt_idx = i * num_classes + y + 1\n        plt.subplot(samples_per_class, num_classes, plt_idx)\n        # plt.imshow(x_train[idx].astype('uint8'))\n        plt.imshow(x_train[idx])\n        plt.axis('off')\n        if i == 0:\n            plt.title(cls)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Adding dimensions for keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.reshape(x_train.shape[0], 28, 28,1)  \nx_val = x_val.reshape(x_val.shape[0], 28, 28,1)  \nx_test = x_test.reshape(x_test.shape[0], 28, 28,1) \nprint(\"Train size:{}\\nvalidation size:{}\\nTest size:{}\".format(x_train.shape,x_val.shape, x_test.shape))\n\nmean_px = x_train.mean().astype(np.float32)\nstd_px = x_train.std().astype(np.float32)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Defining the architecture\n\n### Option 1:\nLight architecture with approximately 50K parameters. \n<br>Dropout for avoiding overfitting\n<br>BatchNormalization for faster convergence time"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.layers import Input , Dense , Conv2D , Activation , Add,ReLU,MaxPool2D,Flatten,Dropout,BatchNormalization\nfrom tensorflow.python.keras.models import Model\n\n\ninput = Input(shape=[28, 28, 1])\nx = Conv2D(32, (5, 5), strides=1, padding='same', name='conv1')(input)\nx = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch1')(x)\nx = Activation('relu', name='relu1')(x)\n# x = Dropout (0.5)(x)\nx = Conv2D(32, (5, 5), strides=1, padding='same', name='conv2')(x)\nx = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch2')(x)\nx = Activation('relu', name='relu2')(x)\n# x = Dropout (0.5)(x)\nx = Conv2D(32, (5, 5), strides=1, padding='same', name='conv2add')(x)\nx = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch2add')(x)\nx = Activation('relu', name='relu2add')(x)\nx = Dropout (0.15)(x)\nx = MaxPool2D(pool_size=2, strides=2, padding='same')(x)\nx = Conv2D(64, (3, 3), strides=1, padding='same', name='conv3')(x)\nx = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch3')(x)\nx = Activation('relu', name='relu3')(x)\nx = Conv2D(64, (3, 3), strides=1, padding='same', name='conv4')(x)\nx = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch4')(x)\nx = Activation('relu', name='relu4')(x)\nx = Conv2D(32, (3, 3), strides=1, padding='same', name='conv5')(x)\nx = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer='uniform',name='batch5')(x)\nx = Activation('relu', name='relu5')(x)\nx = Dropout (0.15)(x)\nx = MaxPool2D(pool_size=2, strides=2)(x)\nx = Flatten()(x)\nx = Dense(100, name='Dense30')(x)\nx = Activation('relu', name='relu6')(x)\nx = Dropout (0.05)(x)\nx = Dense(10, name='Dense10')(x)\nx = Activation('softmax')(x)\nmodel = Model(inputs = input, outputs =x)\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. Train the model using data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.python.keras.callbacks import ReduceLROnPlateau , ModelCheckpoint\nfrom tensorflow.python.keras.optimizers import Adam ,RMSprop\n\ncheckpoint = ModelCheckpoint(\"best_weights.hdf5\", monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n\n# # CREATE MORE IMAGES VIA DATA AUGMENTATION\ndatagen = ImageDataGenerator(\n        rotation_range= 8,  \n        zoom_range = 0.13,  \n        width_shift_range=0.13, \n        height_shift_range=0.13)\n\nepochs = 70\nlr_initial = 0.0011\n# optimizer = RMSprop(lr=0.001, rho=0.95, epsilon=1e-08, decay=0.0)\noptimizer = Adam(lr=lr_initial, decay= lr_initial / (epochs*1.3))\n\nmodel.compile(optimizer=optimizer,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\ndatagen.fit(x_train)\nbatch_size = 64\n\nhistory = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (x_val,y_val),\n                              verbose = 2, steps_per_epoch=x_train.shape[0] // batch_size, callbacks=[checkpoint])\nmodel.load_weights(\"best_weights.hdf5\") \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5. After train visualizations"},{"metadata":{},"cell_type":"markdown","source":"### 5.1 Loss graph visualizations"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(1)\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model Complexity Graph:  Training vs. Validation Loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validate'], loc='upper right')\n\nplt.figure(2)\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('Model Accuracy Graph:  Training vs. Validation accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validate'], loc='upper right')\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5.2 Prediction images visualization\nStright forward taking some images and plotting predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# predicted class\nnum_rows = 6\nnum_cols = 15\nsample_size = num_rows * num_cols\nindices = np.arange(sample_size)\nx_pred = x_test[indices,:,:]\npredictions = model.predict(x_pred)\nx_pred = np.squeeze(x_test[indices,:,:])\ny_pred = np.argmax(predictions,axis=1)\n\nnum_images = num_rows*num_cols\nplt.figure(figsize=(num_cols*2, num_rows*2))\nplt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=0.6)\nfor i in range(num_images):\n  plt.subplot(num_rows, num_cols, i+1)\n  plt.imshow(x_pred[i])\n  plt.title(classes[y_pred[i]])\n  # plt.subplot(num_rows, 2*num_cols, 2*i+2)\n  # plot_value_array(i, predictions, test_labels)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5.3 Confusion matrix\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ny_vecs = model.predict(x_val)\ny_pred = np.argmax(y_vecs, axis=1)\ny_true = y_val\ncm = confusion_matrix(y_true, y_pred)\n# print(cm)\n\n# plt.imshow(cm, cmap = 'ocean')\n# plt.colorbar\n\nmin_val, max_val = 0, 15\n\n# intersection_matrix = np.random.randint(0, 10, size=(max_val, max_val))\nplt.figure(11)\nfig, ax = plt.subplots()\nax.matshow(cm, cmap=plt.cm.Blues)\n# ax.matshow(cm, cmap=plt.cm.magma_r)\n\nfor i in range(10):\n    for j in range(10):\n        c = cm[j,i]\n        ax.text(i, j, str(c), va='center', ha='center')\n\n\nplt.xticks(range(10))\nplt.yticks(range(10))\nplt.title('Confusion matrix',size = 28)\nplt.xlabel('True labeling',size = 20)\nplt.ylabel('Predicted labeling',size = 20)\nplt.rcParams.update({'font.size': 22})\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5.4 Miss-labeled data visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display some error results \n# y_vecs = model.predict(x_test)\n# y_pred = np.argmax(y_vecs, axis=1)\nY_true = y_val\nY_pred_classes =  y_pred\nY_pred = y_vecs\nX_val = x_val\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 2\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=1)\n    plt.figure(figsize=(num_cols, num_rows))\n\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted  :{}\\nTrue  :{}\".format(pred_errors[error],obs_errors[error]), fontsize=14)\n            n += 1\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors \nmost_important_errors = sorted_dela_errors[-25:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict results\nresults = model.predict(x_test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n# results = pd.Series(results,name=\"Label\")\n\n# submission = pd.concat([pd.Series(range(1,4999),name = \"ImageId\"),results],axis = 1)\n\n# submission.to_csv(\"MNIST.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test1 = pd.read_csv('../input/Kannada-MNIST/test.csv').values\nprint(test1)\nx_samples = test1[0:,1:]\nid_samples = test1[0:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict results\nresults = model.predict(x_test)\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\ndg = pd.DataFrame()\ndg['id'] = list(id_samples)\ndg['label'] = results\ndg.to_csv('submission.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(dg.head(3))\n# print(dg.tail(3))\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}