{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Kannada MNIST: Simple CNN"},{"metadata":{},"cell_type":"markdown","source":"### Import Modules"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import initializers, layers\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 18\nnp.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore') # ignore warnings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.test.gpu_device_name() # testing gpu","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"dig_mnist = pd.read_csv(\"../input/Kannada-MNIST/Dig-MNIST.csv\")\nsample_submission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\ntest = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\ntrain = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 28x28 dimension flatten into 784 columns\n* 60000 examples in training\n* First column is the label"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extracting label from training set\nx = train.iloc[:,1:].values\ny = train.label.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y) # Checking if it is homogenous","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It is homogenous, so no need to augment specific labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can just remove the \"id\" columns"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = test.drop(\"id\", axis=1).iloc[:,:].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dig_mnist.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Same process that was used in the training set."},{"metadata":{"trusted":true},"cell_type":"code","source":"x_dig = dig_mnist.drop('label', axis=1).iloc[:,:].values\ny_dig = keras.utils.to_categorical(dig_mnist.label)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We now need to reshape the data from the 784 flat to the 28x28x1."},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = x.reshape(x.shape[0], 28, 28, 1)\ny = keras.utils.to_categorical(y, 10) # and transform y into vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)\nx_dig = x_dig.reshape(x_dig.shape[0], 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training preparation"},{"metadata":{},"cell_type":"markdown","source":"We also need to split our training set into true training and validation. We will use 10% of our dataset as validation.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.1, random_state = seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will use data augmentation to try to increase the performance of our model out of sample. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# This values can change and were inspired by those used in other submissions.\ntrain_datagen = ImageDataGenerator(rescale = 1./255.,\n                                   rotation_range = 10,\n                                   width_shift_range = 0.25,\n                                   height_shift_range = 0.25,\n                                   shear_range = 10,\n                                   zoom_range = 0.1,\n                                   horizontal_flip = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We also need to create one for the validation data set, but this will only rescale the image\nval_datagen = ImageDataGenerator(rescale=1./255.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Different models were tested and inspired by other submissions\nmodel = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(64, (3,3), padding=\"same\", input_shape=(28,28,1)),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(64, (3,3), padding=\"same\"),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    \n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Conv2D(128, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(128, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n   \n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Conv2D(256, (3,3), padding=\"same\"),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(256, (3,3), padding=\"same\"),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    \n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(256),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dense(10, activation=\"softmax\")\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's check our model\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can now define our optimizer and compile the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"initial_learningrate=0.001\nbatch_size = 1024\nepochs = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = RMSprop(learning_rate=initial_learningrate,\n                   momentum=0.1,\n                   centered=True,\n                   name=\"RMSprop\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=\"categorical_crossentropy\", # We will use categorial crossentropy as usual \n              optimizer=optimizer,\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will also define 2 callbacks, one for reducing LR and another for early stopping, both when val_loss is no longer improving."},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor=\"loss\",\n                                            patience=5,\n                                            verbose=1,\n                                            factor=0.2,\n                                            min_lr=0.00001)\n\nes = EarlyStopping(monitor=\"val_loss\", mode=\"min\", verbose=1, patience=20, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are now ready to run our model."},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(\n    train_datagen.flow(x_train, y_train, batch_size=batch_size),\n    steps_per_epoch=len(x_train)//batch_size,\n    epochs=epochs,\n    validation_data=val_datagen.flow(x_val, y_val),\n    validation_steps=50,\n    callbacks=[learning_rate_reduction, es])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy = history.history['accuracy']\nval_accuracy = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs = range(len(accuracy))\nplt.plot(epochs, accuracy, 'b', label='Training accuracy')\nplt.plot(epochs, val_accuracy, 'r', label='Validation accuracy')\nplt.title('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure()\nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_dig = x_dig/255\nx_dig = x_dig.reshape(x_dig.shape[0],28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(x_dig,y_dig,verbose=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict_classes(x_test/255.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['label'] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}