{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt # to plot charts\n\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom sklearn.model_selection import train_test_split\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n# from tensorflow import set_random_seed\n\nseed = 108\nnp.random.RandomState(seed)\n# set_random_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Kers modules\nfrom keras.optimizers import SGD\nfrom keras.models import Sequential\nfrom keras.utils import to_categorical\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers import Dense, Dropout, Conv2D, Flatten, MaxPool2D\nfrom keras.callbacks import EarlyStopping, History, LearningRateScheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load & Prepare the Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# path where the dataset is kept\ndata_dir = \"/kaggle/input/Kannada-MNIST/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read the data from the csv\ntrain_df = pd.read_csv(data_dir+\"train.csv\")\ntrain_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check the number of rows and columns\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract the labels from the dataframe\ny = train_df.values[:, 0]\n\n# convert the y to categorical using one-hot encoding\ny = to_categorical(y)\nprint(\"Shape of y: \", y.shape)\nprint(\"Sample of y: \", y[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract the pixel values from the dataframe\nX = train_df.values[:, 1:]/255.0 # all the columns but 1st\n\n# reshape each row into 28x28 size\nX = X.reshape(-1, 28, 28, 1) # -1 tells the system to automatically figure out the size of the first dimention\n\nprint(\"Shape of X: \", X.shape)\nprint(\"Sample of X: \")\nplt.imshow(X[0].reshape(28, 28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split the data into train(85%) and test (15%)\nvalidation_split = .15\n\n# stratify makes sure that data of all the classes - Tshirt, Trouser, etc. are split equally between train and test\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=validation_split, stratify=y, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train.shape, y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_val.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the network"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNN_Modeling():\n    \n    def __init__(self, model_confs):\n        self.epochs = 30\n        self.batch_size = 80\n        self.annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose=0)\n        \n        self.models = []\n        self.history = []\n        self.model_names = []\n        self.model_confs = model_confs\n    \n    def build_models(self, feature_maps, kernel_size, dense_size, drop_rate):\n        model = Sequential()\n\n        # add the convolution and max pool layers with provided kernel_size\n        for i, fm in enumerate(feature_maps):\n            # add conv layer\n            model.add(Conv2D(fm, kernel_size=kernel_size, padding='same', activation='relu', input_shape=(28, 28, 1)))\n\n            # add MaxPool\n            if i == len(feature_maps)-1:\n                model.add(MaxPool2D())\n            else:\n                model.add(MaxPool2D(padding='same'))\n\n            # add the Dropout\n            model.add(Dropout(drop_rate))\n\n        # convert the output from the convolution layers into a linear array\n        model.add(Flatten())\n\n        # add a dense layer with size - dense_size\n        for dns in dense_size:\n            if dns>0:\n                model.add(Dense(dns, activation='relu'))\n                model.add(Dropout(drop_rate))\n\n        # add the final softmax layer with size equal to the number of categories\n        model.add(Dense(10, activation='softmax'))\n\n        # compile the model\n        model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n        # return the model\n        return model\n    \n    \n    def get_model_name(self, conf):\n        # add the convolution layers name\n        name = \"-\".join(\n            [\"{}C{}P1\".format(ft, conf['kernel_size']) for ft in conf['feature_maps']])\n\n        # add the dense layer size\n        name = name + \"-\" + \"-\".join(map(str, conf['dense_size']))\n\n        # add the drop out\n        name = name + '-D%d'%round(conf[\"drop_rate\"]*100)\n\n        return name    \n    \n    \n    def train_models(self, _x_train, _y_train, _x_val, _y_val):\n        # to store the models and their history\n        self.models = [None]*len(self.model_confs)\n        self.history = [None]*len(self.model_confs)\n        self.model_names = [None]*len(self.model_confs)\n        \n        for i, model_conf in enumerate(self.model_confs):\n            \n            # create the model\n            self.models[i] = self.build_models(**model_conf)\n            \n            # get and store the model name\n            self.model_names[i] = self.get_model_name(model_conf)\n\n            # fir the model\n            self.history[i] = self.models[i].fit(_x_train,_y_train, batch_size=self.batch_size, epochs=self.epochs, \n                          validation_data = (_x_val,_y_val), callbacks=[self.annealer], verbose=0)\n\n            print(\"CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n                self.model_names[i], self.epochs, max(self.history[i].history['accuracy']), \n                max(self.history[i].history['val_accuracy']))\n            )\n            \n            \n    def plot_accuracy_chart(self, accuracy='val_accuracy'):\n        # set the image size\n        plt.figure(figsize=(15,5))\n        \n        # plot the accuracy lines\n        for i in range(len(self.models)):\n            sns.lineplot(\n                x=range(self.epochs),\n                y=self.history[i].history[accuracy], \n                label=self.model_names[i]\n            )\n            \n            \n    def predict(self, model_name, _x_predict):\n        # get the model\n        given_model = self.models[self.model_names.index(model_name)]\n        \n        return given_model.predict_classes(_x_predict)\n    \n    \n    def get_model(self, model_name):\n        # find and return the model\n        return self.models[self.model_names.index(model_name)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Best Model before Data augmentation\nAchieved 98.1% accuracy with the model - [32C5P1]-[64C5P1]-[128C5P1]-256-D30 without data augmentation."},{"metadata":{"trusted":true},"cell_type":"code","source":"# configuration of each of the model\nmodel_confs = [{\"feature_maps\": [32, 64, 128], \"kernel_size\": 5, \"dense_size\": [256], \"drop_rate\": 0.3}]\n\ncnn_model_before_data_aug = CNN_Modeling(model_confs)\ncnn_model_before_data_aug.train_models(X_train, y_train, X_val, y_val)\ncnn_model_before_data_aug.plot_accuracy_chart()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Displaying few misclassified Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the predicted values of the validation dataset\nval_pred_labels = cnn_model_before_data_aug.predict(\"32C5P1-64C5P1-128C5P1-256-D30\", X_val)\n\n# get the labels for the y_val\ny_val_labels = np.argmax(y_val, axis=1)\n\n# misclassifications\nmisclassses = np.where(np.argmax(y_val, axis=1) != val_pred_labels)[0]\n\nprint(\"Number of misclassifications: \", len(misclassses))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nfor i in range(18):\n    plt.subplot(3,6,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X_val[misclassses[i]].reshape(28, 28), cmap=plt.cm.binary)\n    plt.xlabel(\"{} --> {}\".format(y_val_labels[misclassses[i]], val_pred_labels[misclassses[i]]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generating more data by augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# set the configuration of the ImageGenerator\ndatagen = ImageDataGenerator(\n            rotation_range=10,  \n            zoom_range = 0.1,  \n            width_shift_range=0.1, \n            height_shift_range=0.1\n)\n\n# get the best model\nmodel_name = \"32C5P1-64C5P1-128C5P1-256-D30\"\nbes_model_so_far = cnn_model_before_data_aug.get_model(model_name)\n\n# train the best model\nepochs = 50\nhistory = bes_model_so_far.fit_generator(datagen.flow(X_train, y_train, batch_size=64), \n                epochs=epochs, steps_per_epoch=X_train.shape[0]//64,validation_data=(X_val,y_val), \n                callbacks=[cnn_model_before_data_aug.annealer], verbose=0)\n\nprint(\"CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n    model_name,epochs,max(history.history['accuracy']),max(history.history['val_accuracy'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nsns.lineplot(\n    x=range(epochs),\n    y=history.history[\"val_accuracy\"], \n    label=model_name\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_df = pd.read_csv(data_dir+\"test.csv\")\npredict_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the labels of the data\nimg_ids = predict_df.values[:, 0]\n\n# extract the pixel values from the dataframe\nX_predict = predict_df.values[:, 1:]/255.0 # all the columns but 1st\n\n# reshape each row into 28x28 size\nX_predict = X_predict.reshape(-1, 28, 28, 1) # -1 tells the system to automatically figure out the size of the first dimention\n\nprint(\"Shape of X: \", X_predict.shape)\nprint(\"Sample of X: \")\nplt.imshow(X_predict[0].reshape(28, 28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_labels = bes_model_so_far.predict_classes(X_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_prediction = pd.DataFrame()\nfinal_prediction['id'] = img_ids\nfinal_prediction['label'] = predicted_labels\nfinal_prediction.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}