{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Dropout, Dense, Activation, ZeroPadding2D, Flatten\nfrom keras.regularizers import l2\nfrom keras.losses import categorical_crossentropy\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import np_utils\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_layer(model, num_kernels, kernel_size=(3,3), strides=1, activation='relu', pad_mode='same',\n               addBatchNorm=True, addPooling=True):\n    \"\"\"\n    Builds the layers of convolutional neural networks with default (3,3) max pooling.\n\n    :param model: The object of keras.models.Sequential\n    :param num_kernels: Number of kernels\n    :param kernel_size: Size of kernels\n    :param strides: Strides of conv. layer\n    :param activation: Name of activation function\n    :param pad_mode: 'same'/'valid'\n    :param addBatchNorm: True/False\n    :param addPooling: True/False\n    :return: return an object of model.\n    \"\"\"\n    model.add(Conv2D(num_kernels, kernel_size=kernel_size, strides=strides, padding=pad_mode,\n                     kernel_regularizer=l2()))\n    if addBatchNorm:\n        model.add(BatchNormalization(axis=3))\n\n    model.add(Activation(activation))\n\n    if addPooling:\n        model.add(MaxPooling2D((3,3), strides=strides))\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = 10\nbatch_size = 256\nepochs = 200\nimg_x, img_y = (28, 28)\ninput_shape = (img_x, img_y, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"- We combined the **train.csv** and **Dig-MNIST.csv** together as the training dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset1 = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntrain_dataset2 = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')\ntrain_dataset = train_dataset1.append(train_dataset2, ignore_index = True)\ntest_dataset = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\n\ntrain_dataset.groupby(by='label').size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_MNIST = train_dataset.drop(['label'], axis=1).values.reshape(-1, img_x, img_y, 1).astype('float32')/255\ny_MNIST = train_dataset['label']\nx_test = test_dataset.drop([\"id\"], axis=1).values.reshape(-1, img_x, img_y, 1).astype('float32')/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_dev, y_train_origin, y_dev_origin = train_test_split(x_MNIST, y_MNIST, test_size=0.1, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Convert the **y_train** and **y_dev** into one-hot vectors."},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np_utils.to_categorical(y_train_origin, num_classes=num_classes)\ny_dev = np_utils.to_categorical(y_dev_origin, num_classes=num_classes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Learning rate decay."},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_reduction = ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=5, factor=np.sqrt(0.5), verbose=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Image data generator."},{"metadata":{"trusted":true},"cell_type":"code","source":"img_gen = ImageDataGenerator( featurewise_center=False,\n                              samplewise_center=False,\n                              featurewise_std_normalization=False,\n                              samplewise_std_normalization=False,\n                              zca_whitening=False,\n                              rotation_range=10,\n                              zoom_range=0.10,\n                              width_shift_range=0.1,\n                              height_shift_range=0.1,\n                              horizontal_flip=False,\n                              vertical_flip=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Construct model\n- Majorly, we use continuously conv_layer function to stack series **CONV layers** with the same number of kernels and that was called a group. And we connected the each group by a **Dropout layer**.\n- Finally, 2 **Fully connected layers** were set behind the **Flatten**."},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(ZeroPadding2D((1,1), input_shape=input_shape))\n# Group 1\nmodel = conv_layer(model, 32, kernel_size=(5, 5), strides=1, activation='relu', pad_mode='valid', addPooling=True)\nmodel = conv_layer(model, 32, kernel_size=(5, 5), strides=1, activation='relu', pad_mode='same', addPooling=True)\nmodel = conv_layer(model, 32, kernel_size=(5, 5), strides=1, activation='relu', pad_mode='valid', addPooling=True)\nmodel.add(Dropout(0.2))\n# Group 2\nmodel = conv_layer(model, 64, kernel_size=(3, 3), strides=1, activation='relu', pad_mode='valid', addPooling=True)\nmodel = conv_layer(model, 128, kernel_size=(3, 3), strides=1, activation='relu', pad_mode='same', addPooling=False)\nmodel = conv_layer(model, 128, kernel_size=(3, 3), strides=1, activation='relu', pad_mode='same', addPooling=False)\nmodel = conv_layer(model, 128, kernel_size=(3, 3), strides=1, activation='relu', pad_mode='valid', addPooling=False)\nmodel.add(Dropout(0.2))\n# Group 3\nmodel = conv_layer(model, 256, kernel_size=(3, 3), strides=1, activation='relu', pad_mode='valid', addPooling=True)\n\nmodel.add(Flatten())\nmodel.add(Dropout(0.2))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dense(num_classes, activation='softmax'))\n\nmodel.compile(optimizer=Adam(),\n              loss=categorical_crossentropy,\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Print information about this model and the data."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model.summary())\nprint('Size of training dataset:', x_train.shape[0])\nprint('Size of dev. dataset:    ', x_dev.shape[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Start training."},{"metadata":{"trusted":true},"cell_type":"code","source":"img_gen.fit(x_train)\nmodel.fit_generator(img_gen.flow(x_train, y_train, batch_size=batch_size),\n                    epochs=epochs,\n                    steps_per_epoch=x_train.shape[0] // batch_size,\n                    verbose=2,\n                    validation_data=(x_dev, y_dev),\n                    callbacks=[lr_reduction])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluate model by testing dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('\\nPredicting...')\npredictions = model.predict(x_test)\nprint('Prediction completed.')\npredictions = np.argmax(predictions, axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Save result as CSV for submission."},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('07.Kaggle_submission.csv', 'w', newline='') as csv_file:\n    print('\\nSaving file...')\n    csv_writer = csv.writer(csv_file, delimiter=',')\n    # Define column name.\n    csv_writer.writerow(['id', 'label'])\n    for i in range(len(predictions)):\n        csv_writer.writerow([i, predictions[i]])\n\n    print('File: 07.Kaggle_submission.csv Saved completed.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix\n- We not only evaluate the performance through the accuracy in developing phase but also use the confusion matrix might be more specific this model would be trained becoming overfitting or not."},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sn\nimport matplotlib.pyplot as plt\n\ndef plot_confusion_matrix(cm, num_classes, title='Confusion matrix'):\n    print('Title: ', title)\n    print(cm)\n    \n## Show confusion matrix\n# Training dataset\npredict_train = model.predict(x_train)\n# Convert the one-hot vectors to the corresponding classes.\npredict_train = np.argmax(predict_train, axis=1)\ny_train = np.argmax(y_train, axis=1)\n# Build confusion matrix.\nconfusion_mx = confusion_matrix(y_train, predict_train)\nplot_confusion_matrix(confusion_mx, num_classes=num_classes, title='Confusion matrix - training set')\n\n# Developing dataset\npredict_dev = model.predict(x_dev)\n# Convert the one-hot vectors to the corresponding classes.\npredict_dev = np.argmax(predict_dev, axis=1)\ny_dev = np.argmax(y_dev, axis=1)\n# Build confusion matrix.\nconfusion_mx = confusion_matrix(y_dev, predict_dev)\nplot_confusion_matrix(confusion_mx, num_classes=num_classes, title='Confusion matrix - developing set')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}