{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Step 1: Prepare (images, label) split for train, test, validation sets"},{"metadata":{"id":"wYtuKeK0dImp","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"id":"4kxw-_rmcnVu","outputId":"44e2946b-72aa-43f5-eeab-5727dfe436a3","trusted":true},"cell_type":"code","source":"# read data from csv file\ndef get_data(file):\n    data = pd.read_csv(file)\n    data = data.to_numpy()\n    labels = []\n    images = []\n    \n    for row_ind in range(data.shape[0]):\n        labels.append(data[row_ind, 0])\n        images.append(data[row_ind, 1:])\n        \n    images = np.reshape(images, newshape=(-1, 28, 28))\n    labels = np.array(labels)\n    return images, labels\n\nX_train, y_train = get_data('../input/Kannada-MNIST/train.csv')\nX_test, test_Id = get_data('../input/Kannada-MNIST/test.csv')\n\nprint(X_train.shape)\nprint(y_train.shape)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(y_train).value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Evenly distributed, non-skewed classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n# make sure train and val are from same distribution whether is class counts or image-label quality\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.15, random_state=42, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"IEc6z0co-XYQ","outputId":"0f27dd4a-f187-4859-db5e-9e7fb6d8e31e","trusted":true},"cell_type":"code","source":"# visualization of one random data sample\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimgplot = plt.imshow(X_train[24], cmap='gray')\nprint(y_train[24])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 2: One-hot encoder and ImageDataGenerator for image augmentation"},{"metadata":{"id":"OWH-L245W3xE","outputId":"d63d4569-4706-4e8c-af19-c44f4d43a64b","trusted":true},"cell_type":"code","source":"# use one-hot encoder for multi-class classification\nfrom keras.utils import to_categorical\ny_train = to_categorical(y_train)\ny_val = to_categorical(y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add another dimension to the data since they are gray scale images\nX_train = np.expand_dims(X_train, axis=3)\nX_val = np.expand_dims(X_val, axis=3)\nX_test = np.expand_dims(X_test, axis=3)","execution_count":null,"outputs":[]},{"metadata":{"id":"awoqRpyZdQkD","outputId":"844eab4b-55ac-4c1e-8dcb-937810276f8b","trusted":true},"cell_type":"code","source":"# do image augmentation\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = ImageDataGenerator(\n        rotation_range=10, \n        zoom_range = 0.1,  \n        width_shift_range=0.1, \n        height_shift_range=0.1) \n\nprint(X_train.shape)\nprint(X_val.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 3: Define simple CNN baseline model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\ntf.keras.backend.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Input, Conv2D, BatchNormalization, ReLU, MaxPooling2D, AveragePooling2D, Flatten, Dense\nfrom keras.models import Model\n\ninputs = Input(shape=(28, 28, 1))\n\nx = Conv2D(32, (3, 3), padding='same', activation='relu')(inputs)\nx = BatchNormalization(momentum=0.1)(x)\nx = Conv2D(32, (3, 3), padding='same', activation='relu')(x)\nx = BatchNormalization(momentum=0.1)(x)\nx = AveragePooling2D((2, 2))(x)\n\nx = Conv2D(16, (5, 5), padding='same', activation='relu')(x)\nx = BatchNormalization(momentum=0.1)(x)\nx = Conv2D(16, (5, 5), padding='same', activation='relu')(x)\nx = BatchNormalization(momentum=0.1)(x)\nx = AveragePooling2D((2, 2))(x)\n\nx = Flatten()(x)\n# from feature representation to categorical prob\nx = Dense(100, activation='sigmoid')(x)\nx = BatchNormalization(momentum=0.1)(x)\npred = Dense(10, activation='softmax')(x)\n\nmodel = Model(inputs=inputs, outputs=pred)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"id":"Rmb7S32cgRqS","outputId":"4b65ae3a-6a8c-4c78-f6ee-e5b6bb36e928","trusted":true},"cell_type":"code","source":"from keras.optimizers import Adam\nadam = Adam(learning_rate=0.00005)\n\n# Compile Model\nmodel.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n\nfrom tensorflow.keras.callbacks import EarlyStopping\n# setup callbacks for auto earlystopping\nes = EarlyStopping(monitor='val_accuracy', min_delta=0.00001, patience=5, verbose=1, mode='auto', restore_best_weights=True)\n\n# Train the Model\nbatch_size = 64\nepochs = 50\nhistory = model.fit_generator(train_datagen.flow(X_train, y_train, batch_size=batch_size), \n                              epochs=epochs,\n                              steps_per_epoch=X_train.shape[0]//batch_size,\n                              validation_data=(X_val, y_val),\n                              callbacks=[es],\n                              verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"_Q3Zpr46dsij","trusted":true},"cell_type":"code","source":"# Plot the chart for accuracy and loss on both training and validation\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'r', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\n\n# creates a new figure\nplt.figure()\nplt.plot(epochs, loss, 'r', label='Training Loss')\nplt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 4: Error analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"# confusion matrix for validation data\nfrom sklearn.metrics import confusion_matrix\nval_prediction = model.predict(X_val)\nval_prediction = np.argmax(val_prediction, axis=1)\n\nval_truth = np.argmax(y_val,axis=1)\n# conf[i, j] is no. of true i are predicted to be j\nconf = confusion_matrix(val_truth, val_prediction)\nconf = pd.DataFrame(conf, index=range(0,10), columns=range(0,10))\nfrom IPython.display import display\ndisplay(conf)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most of the wrong prediction comes some of the true 0s are predicted to be 1s, some of true 3s are predicted to be 7s, lets see why this is the case below."},{"metadata":{},"cell_type":"markdown","source":"Kannada numbers official representation\n![](https://t3.ftcdn.net/jpg/01/62/22/58/500_F_162225807_ntQwmEkc76NoItsdipA1eNHQZKqdBoio.jpg)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# analyse how the majority wrong classification is caused\ndef case_analysis():\n    for i in range(len(val_prediction)):\n        if (val_prediction[i] == 1 and val_truth[i] == 0) or (val_prediction[i] == 7 and val_truth[i] == 3):\n            yield i\n\ncases = list(case_analysis())  \nprint('no. of cases: {}'.format(len(cases)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(20, 20))\ncolumns = 5\nrows = 3\nfor i in range(1, columns*rows +1):\n    if i == 14:\n        break\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(np.squeeze(X_val[cases[i-1]]))\n    plt.title('pred: {}, truth: {}'.format(val_prediction[cases[i-1]], val_truth[cases[i-1]]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As one can see from the above, almost all of the 'wrongly classified case' are due to wrong labels, for example, for the 'pred 1, truth 0' cases, human can tell it is obviously a 1 rather than 0 as the ground truth, the similar goes for 7 and 3. This is rather unfortunate due to human labelling during data preparation. Therefore, as far as this validation performance of our model, it is as perfect and as simple as a baseline model can get."},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate prediction for test set\nprediction = model.predict(X_test)\nprediction = np.argmax(prediction, axis=1)\nsubmission_df = pd.DataFrame({'id':test_Id , 'label':prediction.astype(np.int32)})\n# for kernel based competition\nsubmission_df.to_csv('./submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Exercise 8 - Question.ipynb","version":"0.3.2","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":4}