{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false,"_kg_hide-output":true},"cell_type":"markdown","source":"## Kannada digits recognition with Keras Functional API\nThis is my first keras project. NN model is not created by me.\nI use one from Happy House exercise from Andrew Ng course on coursera.org\nI tried to modify it, but it looks like results are best as is.\n\nThis notebook consists of these parts:\n\n1. Data download.\n2. Data estimation. Looking in details what we have to work with.\n3. Data augmentation, preparing training/validation sets.\n4. NN model.\n5. Selecting hyperparameters for training, compiling model\n6. Model training and evaluating\n7. Analyzing mislabeled data.\n8. Submitting results\n\nYou are welcome, upvote if you want to support me!"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nfrom keras.models import Model, load_model, save_model\nfrom keras.layers import Conv2D, Dense, Input, Activation, BatchNormalization, Dropout, Flatten\nfrom keras.layers import MaxPooling2D\nfrom keras.optimizers import Adam","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1. Data download."},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest_csv = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Data estimation. Looking in details what we have to work with."},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_csv.label.values\nx = train_csv.drop(columns=['label']).values\nprint('x shape: ', x.shape)\nprint('y shape: ', y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = x.reshape(60000,28,28)\nplt.figure()\nplt.imshow(x[0], cmap=plt.cm.binary)\nplt.colorbar()\nplt.grid(False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#let's select 5 random images from each class\nnp.random.seed(2)\nplt.figure(figsize=(15,15))\nfor label in range(10):\n    for i,n in enumerate(train_csv.loc[train_csv.label == label].sample(5).index):\n        plt.subplot(10,5,(label*5+i+1))\n        plt.xticks([])\n        plt.yticks([])\n        plt.grid(False)\n        plt.imshow(x[n], cmap=plt.cm.binary)\n        plt.xlabel(y[n])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. Data augmentation, preparing training/validation sets.\nI decided not to use ImageGenerator in this project, as we have sufficient number of examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"# take random 5000 from train_csv for later accuracy validation\n# else goes to training\nval_df = train_csv.sample(5000)\ntrain_df = train_csv.drop(index=val_df.index)\n\n# prepare data for training\n# set input shape for Conv2D layers\nx_train = train_df.drop(columns=['label']).values\nx_train = x_train.reshape(55000,28,28,1)\n\ny_train = train_df.label.values\n\nx_val = val_df.drop(columns=['label']).values\nx_val = x_val.reshape(5000,28,28,1)\n\ny_val = val_df.label.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. NN model."},{"metadata":{"trusted":true},"cell_type":"code","source":"def main_model(input_shape):\n    \"\"\"\n    Current NN model\n    Arguments:\n        input_shape -- shape of the images of the dataset\n    Returns:\n        model -- a Model() instance in Keras\n    \"\"\"\n    \n    X_input = Input(input_shape)\n    X = Conv2D(32, (3, 3), strides = (1, 1), name = 'conv2d0', padding=\"same\")(X_input)\n    X = BatchNormalization(name = 'bn0')(X)\n    X = Activation('relu')(X)\n    \n    X = MaxPooling2D((2, 2), name='max_pool0')(X)\n    \n    X = Conv2D(128, (1, 1), strides = (1, 1), name = 'conv2d1', padding=\"same\")(X) \n    X = Activation('relu')(X)\n    \n    X = Conv2D(32, (3, 3), strides = (1, 1), name = 'conv2d2', padding=\"same\")(X)\n    X = BatchNormalization(axis = 3, name = 'bn1')(X)\n    X = Activation('relu')(X)\n    \n    X = MaxPooling2D((2, 2), name='max_pool1')(X)\n    \n    X = Flatten()(X)\n    X = Dropout(rate = 0.5)(X)\n    \n    X = Dense(64, activation='relu', name='fc0')(X)\n    \n    X = Dense(10, activation='softmax', name='fc1')(X)\n\n    return Model(inputs = X_input, outputs = X, name='142K_Conv_NN')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5. Selecting hyperparameters for training, compiling model\nI experimented with different parameters and desided these are the best:\n\n- batch_size = 20 - small is good for accuracy but bad for training time.\n- optimizer = Adam\n- learning_rate = 0.001 - default is the best"},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = main_model(x_train[0].shape)\nmodel1.compile(optimizer = \"Adam\", loss = \"sparse_categorical_crossentropy\", metrics = [\"accuracy\"])\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 6. Model training and evaluating"},{"metadata":{"trusted":true},"cell_type":"code","source":"result = model1.fit(x = x_train, y = y_train, batch_size=20, epochs=15, validation_split=0.1, verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(121)\nplt.plot(result.history['accuracy'], label='acc')\nplt.plot(result.history['val_accuracy'], label='val_acc')\nplt.title('Validation accuracy')\nplt.title('Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(loc='lower right')\nplt.subplot(122)\nplt.plot(result.history['loss'], label='loss')\nplt.plot(result.history['val_loss'], label='val_loss')\nplt.title('Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(loc='upper right')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = model1.evaluate(x = x_val, y = y_val)\nprint (\"Evaluation Set Accuracy = \" + str(preds[1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 7. Analyzing mislabeled data.\nLet's have a look on mislabeled predictions in our validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model1.predict(x_val)\ntest_labels = y_val\ntest_images = x_val.squeeze()\n\ntest_labels = pd.Series(y_val)\npred_labels = pd.Series(predictions.argmax(axis=1))\n\n# sample 15 random mislabeled images\nwrong_class = list( test_labels[test_labels != pred_labels].sample(15).index )\nprint(\"Incorrect classification indexes: \", wrong_class)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"some functions for drawing:"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = [0,1,2,3,4,5,6,7,8,9,10]\ndef plot_image(i, predictions_array, true_label, img):\n    predictions_array, true_label, img = predictions_array, true_label[i], img[i]\n    plt.grid(False)\n    plt.xticks([])\n    plt.yticks([])\n\n    plt.imshow(img, cmap=plt.cm.binary)\n\n    predicted_label = np.argmax(predictions_array)\n    if predicted_label == true_label:\n      color = 'blue'\n    else:\n      color = 'red'\n\n    plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n                                100*np.max(predictions_array),\n                                class_names[true_label]),\n                                color=color)\n\ndef plot_value_array(i, predictions_array, true_label):\n    predictions_array, true_label = predictions_array, true_label[i]\n    plt.grid(False)\n    plt.xticks(range(10))\n    plt.yticks([])\n    thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n    plt.ylim([0, 1])\n    predicted_label = np.argmax(predictions_array)\n\n    thisplot[predicted_label].set_color('red')\n    thisplot[true_label].set_color('blue')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cols = 3\nnum_rows = 5\nplt.figure(figsize=(2*2*num_cols, 2*num_rows))\nfor i, idx in enumerate(wrong_class):\n    #print(num_rows, num_cols, 2*i+1)\n    plt.subplot(num_rows, 2*num_cols, 2*i+1)\n    plot_image(idx, predictions[idx], test_labels, test_images)\n    #print(num_rows, num_cols, 2*i+2)\n    plt.subplot(num_rows, 2*num_cols, 2*i+2)\n    plot_value_array(idx, predictions[idx], test_labels)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 8. Submitting results"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x = test_csv.drop(columns=['id']).values\nprint('test_x shape: ', test_x.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x = test_x.reshape(test_x.shape[0],28,28,1)\npredictions = model1.predict(test_x)\n\npred_labels = predictions.argmax(axis=1)\nsubmission = pd.DataFrame({'id': test_csv['id'], 'label': pred_labels})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}