{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# importing visualization libraries\n\n%matplotlib inline\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading the training data\n\ntrain_df = pd.read_csv(\"/kaggle/input/Kannada-MNIST/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading the testing data\n\ntest_df = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dimensions of training data\n\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dimensions of testing data\n\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# basic meta data about training data \n\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# basic meta data about test data\n\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# distribution of the target label\n\ntrain_df['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualizing the target variable distribution from the training dataset\n\ntrain_df['label'].value_counts().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Building a Model (Y network) using Keras CNN Conv2D (Functional API)\n\n1. We'll use Keras CNN 2D, to create Left and Right network of our Y network (i.e., Left and Right branches). Once it's created then we'll concatenate and send output to a Dense Network.\n2. Try to fit the above model using the train and validation data and save the best model.\n3. Using the best model saved, try to predict the target label of the test data and submit the same."},{"metadata":{"trusted":true},"cell_type":"code","source":"# importing keras libraries\nimport keras\nimport sklearn\n\nfrom keras.preprocessing import image\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Flatten, BatchNormalization, Activation, Dropout, LeakyReLU, Input, Conv2D, MaxPooling2D\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfrom keras import optimizers\nfrom keras.layers.merge import concatenate\nfrom keras import utils\nfrom keras.utils import plot_model\n\n# Splitting the data into train and test\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###### Model Building Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of images per batch\nBATCH_SIZE = 64\n\n# number of epochs or no.f times the data has to be projected towards model\nNP_EPOCHS = 50\n\n# number of output classes\nNP_CLASSES  = 10\n\n# split the data into train and validation use 20% of data for validation\nVALIDATION_SPLIT = 0.20\n\n# optimizer has been passed to the model\nOPTIMIZER = optimizers.RMSprop()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dividing the training data into train and target\n\n# training data\ntrain = train_df.drop('label', axis=1)\n\n# target data\ntarget = train_df['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test data\n\ntest = test_df.drop('id', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample target values\n\ntarget.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Normalizing the training data and test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalizing the train data\n\ntrain /= 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalizing the test data\n\ntest /= 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reshaping the training data\n\ntrain = train.values.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# taking a sample image from traing data\n\nplt.imshow(train[42][:, :, 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reshaping the test data\n\ntest = test.values.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# taking a sample image from test data\n\nplt.imshow(test[42][:, :, 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# converting the target value into categorical values\n\ntarget = utils.to_categorical(target, NP_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample target encoded values\n\ntarget[:2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Model Construction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Output activation function\nNP_OUTPUT_FUNC = \"softmax\"\n\n# Neuron activation fuction\nNP_DESCRIMINATOR_FUNC = \"sigmoid\"\n\n# 15% Drouput or Dropping the signal or connection between the layers\nNP_DROPOUT_FIRST = 0.15\n\n# 10% Dropout or Dropping the signal or connetion between the layers\nNP_DROPOUT_SECOND = 0.10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Input shape\ninput_shape = (28, 28, 1)\n\n# Kernel Size\nNP_KERNEL_SIZE = 3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Creating the Left branch of Y network"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code to create Left branch of the network\nleft_input = Input(shape=input_shape)\nx = left_input\nNP_FILTERS = 32\n\nfor i in range(3):\n    x = Conv2D(filters=NP_FILTERS, kernel_size=NP_KERNEL_SIZE, padding=\"same\")(x)\n    x = LeakyReLU(alpha=0.15)(x)\n    x = BatchNormalization(axis=-1, momentum=0.9, epsilon=0.001, center=True,\n                           scale=True, beta_initializer=\"zeros\", gamma_initializer=\"ones\",\n                           moving_mean_initializer=\"zeros\", moving_variance_initializer=\"one\",\n                           beta_regularizer=None, gamma_regularizer=None,\n                           beta_constraint=None, gamma_constraint=None)(x)\n    x = Conv2D(filters=NP_FILTERS, kernel_size=NP_KERNEL_SIZE)(x)\n    x = LeakyReLU(alpha=0.15)(x)\n    x = BatchNormalization(axis=-1, momentum=0.9, epsilon=0.001, center=True,\n                           scale=True, beta_initializer=\"zeros\", gamma_initializer=\"ones\",\n                           moving_mean_initializer=\"zeros\", moving_variance_initializer=\"one\",\n                           beta_regularizer=None, gamma_regularizer=None,\n                           beta_constraint=None, gamma_constraint=None)(x)\n    x = Dropout(NP_DROPOUT_FIRST)(x)\n    x = MaxPooling2D()(x)\n    NP_FILTERS *= 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Creating the Right branch of the Y network"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code to create Right branch of the network\n\nright_input = Input(shape=input_shape)\ny = right_input\nNP_FILTERS = 32\n\nfor i in range(3):\n    y = Conv2D(filters=NP_FILTERS, kernel_size=NP_KERNEL_SIZE, padding=\"same\")(y)\n    y = LeakyReLU(alpha=0.15)(y)\n    y = BatchNormalization(axis=-1, momentum=0.9, epsilon=0.001, center=True,\n                           scale=True, beta_initializer=\"zeros\", gamma_initializer=\"ones\",\n                           moving_mean_initializer=\"zeros\", moving_variance_initializer=\"one\",\n                           beta_regularizer=None, gamma_regularizer=None,\n                           beta_constraint=None, gamma_constraint=None)(y)\n    y = Conv2D(filters=NP_FILTERS, kernel_size=NP_KERNEL_SIZE)(y)\n    y = LeakyReLU(alpha=0.15)(y)\n    y = BatchNormalization(axis=-1, momentum=0.9, epsilon=0.001, center=True,\n                           scale=True, beta_initializer=\"zeros\", gamma_initializer=\"ones\",\n                           moving_mean_initializer=\"zeros\", moving_variance_initializer=\"one\",\n                           beta_regularizer=None, gamma_regularizer=None,\n                           beta_constraint=None, gamma_constraint=None)(y)\n    y = Dropout(NP_DROPOUT_FIRST)(y)\n    y = MaxPooling2D()(y)\n    NP_FILTERS *= 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Concatinating both Left and Right branches of the Y network\n\nnetwork = concatenate([x, y])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sending the the above concatenated result to a Dense network\n\nnetwork = Flatten()(network)\nnetwork = Dense(512)(network)\nnetwork = LeakyReLU(alpha=0.10)(network)\nnetwork = BatchNormalization(axis=-1, momentum=0.9, epsilon=0.001, center=True, scale=True, beta_initializer=\"zeros\",\n                             gamma_initializer=\"ones\", moving_mean_initializer=\"zeros\",\n                             moving_variance_initializer=\"one\", beta_regularizer=None, gamma_regularizer=None,\n                             beta_constraint=None, gamma_constraint=None)(network)\nnetwork = Dropout(NP_DROPOUT_SECOND)(network)\n\nnetwork = Dense(256)(network)\nnetwork = LeakyReLU(alpha=0.10)(network)\nnetwork = BatchNormalization(axis=-1, momentum=0.9, epsilon=0.001, center=True,\n                             scale=True, beta_initializer=\"zeros\", gamma_initializer=\"ones\",\n                             moving_mean_initializer=\"zeros\", moving_variance_initializer=\"one\",\n                             beta_regularizer=None, gamma_regularizer=None,\n                             beta_constraint=None, gamma_constraint=None)(network)\nnetwork = Dropout(NP_DROPOUT_SECOND)(network)\n\noutput = Dense(NP_CLASSES)(network)\noutput = Activation(NP_OUTPUT_FUNC)(output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Constructing the model \n\nmodel = Model([left_input, right_input], output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Summary of the model\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plotting the model\n\nplot_model(model, to_file=\"y-network.png\", show_shapes=True, show_layer_names=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating optimizer: Using RMSProp optimizer\n\n# optimizer = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)\noptimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, decay=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compiling the model\n\n# compiling the model\n\nmodel.compile(\n    optimizer=optimizer,\n    loss=\"categorical_crossentropy\",\n    metrics=[\"categorical_accuracy\"]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Splitting the training data into train and validation using sklearn libraries\n\nX_train, X_val, y_train, y_val = train_test_split(train, target, test_size=0.20, random_state=123456789)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"X training data shape: {}\".format(X_train.shape))\nprint(\"Y training data shape: {}\".format(y_train.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"X validation data shape: {}\".format(X_val.shape))\nprint(\"Y validation data shape: {}\".format(y_val.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating a generator to get the data as small batches in a lazy format - Data augumentation\n\ngenerator = image.ImageDataGenerator(\n    featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=10,\n    zoom_range=0.1,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    vertical_flip=False,\n    horizontal_flip=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_train_sequences = len(X_train)\nnum_val_sequences = len(X_val)\n\nprint(\"# training sequences: {}\".format(num_train_sequences))\nprint(\"# validation sequences: {}\".format(num_val_sequences))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculating number of training and validation steps per epoch\n# for training\nif (num_train_sequences % BATCH_SIZE) == 0:\n    steps_per_epoch = int(num_train_sequences / BATCH_SIZE)\nelse:\n    steps_per_epoch = int(num_train_sequences / BATCH_SIZE) + 1\n    \n# for validation    \nif (num_val_sequences % BATCH_SIZE) == 0:\n    validation_steps = int(num_val_sequences / BATCH_SIZE)\nelse:\n    validation_steps = int(num_val_sequences / BATCH_SIZE) + 1    \n    \nprint(\"# number of steps required for training: {}\".format(steps_per_epoch))\nprint(\"# number of steps required for validation: {}\".format(validation_steps))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Creating Call backs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# importing datatime package\nimport datetime\n\ncurrent_dt_time = datetime.datetime.now()\nmodel_name = 'model_init' + '_' + str(current_dt_time).replace(' ', '').replace(':', '_') + '/'\n\nif not os.path.exists(model_name):\n    os.mkdir(model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call back: To save the best model based on validation categorical accuracy score\n\nfile_path = model_name + \"model-{epoch:05d}-{loss:.5f}-{categorical_accuracy:.5f}-{val_loss:.5f}-{val_categorical_accuracy:.5f}.h5\"\ncheckpoint = ModelCheckpoint(filepath=file_path, monitor='val_categorical_accuracy', verbose=1, save_best_only=True,\n                             save_weights_only=False, mode='auto', period=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call back: ReduceOnPlateau - If a plateau found\n \nLR = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.2, patience=2, min_lr=0.000001, verbose=1, cooldown=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call back: Exponential Decay\n\ndef exponential_decay_fn(epoch):\n    return 0.01 * 0.1**(epoch / 20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# call back: Exponential Decay\n\ndef exponential_decay(lr, s):\n    def exponential_decay_fn(epoch):\n        return lr * 0.1**(epoch / s)\n    return exponential_decay_fn\n\nexponential_decay_fn = exponential_decay(lr=0.001, s=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_scheduler = keras.callbacks.LearningRateScheduler(exponential_decay_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# adding all callbacks\n\ncallbacks = [\n    # Model checkpoint\n    checkpoint, \n    # ReduceOnPlateau\n    LR, \n    # Learning Rate Scheduler\n    lr_scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fitting the model\nVERBOSE = 1\nhistory = model.fit_generator(generator.flow([X_train, X_train], y_train, batch_size=BATCH_SIZE), \n                             validation_data=generator.flow([X_val, X_val], y_val, batch_size=BATCH_SIZE),\n                             epochs=NP_EPOCHS,\n                             verbose=VERBOSE,\n                             steps_per_epoch=steps_per_epoch,\n                             validation_steps=validation_steps,\n                             class_weight=None,\n                             initial_epoch=0,\n                             callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# finding the best model\n\n# best accuracy found\n# best model: saving model to model_init_2019-07-0108_08_42.188814/model-00009-0.21558-0.93310-0.12834-0.96155.h5\n\nvalues = {}\nmodels = os.listdir(model_name)\n\nfor model in models:\n    converted = model.replace(\".h5\", \"\")\n    accuracy = float(converted.split(\"-\")[-1])\n    values.update({accuracy: model})\n    \nkey = max(values, key = values.get)\nbest = values.get(key)\n\n# Best model found among all the saved models\nprint(\"Best model found: {}\".format(best))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# stored history information\n\nhistory.history.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['categorical_accuracy'])\nplt.plot(history.history['val_categorical_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# summarize history for loss\n\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading the above found model\nfrom keras.models import load_model\n\nmodel_path = model_name + best\nprint(\"Absolute path found: {}\".format(model_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading the model\n\nmodel = load_model(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# summary of the loaded model\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predicting on test data\n\npredictions = model.predict([test, test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample predictions\n\npredictions[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# converting the above prediction into target classes\n\nconversion = np.argmax(predictions, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# converting into series\n\nresult = pd.Series(conversion, name=\"label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# total length of result\n\nprint(\"No.f Id's available in the test dataset: {}\".format(len(test_df['id'])))\nprint(\"No.f predictions done by the model on test data: {}\".format(len(result)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating a dataframe called submission to submit the data\n\nsubmission = pd.concat([test_df['id'], result],axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating submission file\n\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}