{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt # to plot charts\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n# from tensorflow import set_random_seed\n\nseed = 108\nnp.random.RandomState(seed)\n# set_random_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Kers modules\nfrom keras.optimizers import SGD\nfrom keras.models import Sequential\nfrom keras.utils import to_categorical\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers import Dense, Dropout, Conv2D, Flatten, MaxPool2D\nfrom keras.callbacks import EarlyStopping, History, LearningRateScheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load & Prepare the Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# path where the dataset is kept\ndata_dir = \"/kaggle/input/Kannada-MNIST/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read the data from the csv\ntrain_df = pd.read_csv(data_dir+\"train.csv\")\ntrain_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check the number of rows and columns\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract the labels from the dataframe\ny = train_df.values[:, 0]\n\n# convert the y to categorical using one-hot encoding\ny = to_categorical(y)\nprint(\"Shape of y: \", y.shape)\nprint(\"Sample of y: \", y[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract the pixel values from the dataframe\nX = train_df.values[:, 1:]/255.0 # all the columns but 1st\n\n# reshape each row into 28x28 size\nX = X.reshape(-1, 28, 28, 1) # -1 tells the system to automatically figure out the size of the first dimention\n\nprint(\"Shape of X: \", X.shape)\nprint(\"Sample of X: \")\nplt.imshow(X[0].reshape(28, 28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split the data into train(70%) and test (30%)\nvalidation_split = .3\n\n# stratify makes sure that data of all the classes - Tshirt, Trouser, etc. are split equally between train and test\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=validation_split, stratify=y, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train.shape, y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_val.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the network"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNN_Modeling():\n    \n    def __init__(self, model_confs):\n        self.epochs = 30\n        self.batch_size = 80\n        self.annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose=0)\n        \n        self.models = []\n        self.history = []\n        self.model_names = []\n        self.model_confs = model_confs\n    \n    def build_models(self, feature_maps, kernel_size, dense_size, drop_rate):\n        model = Sequential()\n\n        # add the convolution and max pool layers with provided kernel_size\n        for i, fm in enumerate(feature_maps):\n            # add conv layer\n            model.add(Conv2D(fm, kernel_size=kernel_size, padding='same', activation='relu', input_shape=(28, 28, 1)))\n\n            # add MaxPool\n            if i == len(feature_maps)-1:\n                model.add(MaxPool2D())\n            else:\n                model.add(MaxPool2D(padding='same'))\n\n            # add the Dropout\n            model.add(Dropout(drop_rate))\n\n        # convert the output from the convolution layers into a linear array\n        model.add(Flatten())\n\n        # add a dense layer with size - dense_size\n        for dns in dense_size:\n            if dns>0:\n                model.add(Dense(dns, activation='relu'))\n                model.add(Dropout(drop_rate))\n\n        # add the final softmax layer with size equal to the number of categories\n        model.add(Dense(10, activation='softmax'))\n\n        # compile the model\n        model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n        # return the model\n        return model\n    \n    \n    def get_model_name(self, conf):\n        # add the convolution layers name\n        name = \"-\".join(\n            [\"{}C{}P1\".format(ft, conf['kernel_size']) for ft in conf['feature_maps']])\n\n        # add the dense layer size\n        name = name + \"-\" + \"-\".join(map(str, conf['dense_size']))\n\n        # add the drop out\n        name = name + '-D%d'%round(conf[\"drop_rate\"]*100)\n\n        return name    \n    \n    \n    def train_models(self, _x_train, _y_train, _x_val, _y_val):\n        # to store the models and their history\n        self.models = [None]*len(self.model_confs)\n        self.history = [None]*len(self.model_confs)\n        self.model_names = [None]*len(self.model_confs)\n        \n        for i, model_conf in enumerate(self.model_confs):\n            \n            # create the model\n            self.models[i] = self.build_models(**model_conf)\n            \n            # get and store the model name\n            self.model_names[i] = self.get_model_name(model_conf)\n\n            # fir the model\n            self.history[i] = self.models[i].fit(_x_train,_y_train, batch_size=self.batch_size, epochs=self.epochs, \n                          validation_data = (_x_val,_y_val), callbacks=[self.annealer], verbose=0)\n\n            print(\"CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n                self.model_names[i], self.epochs, max(self.history[i].history['accuracy']), \n                max(self.history[i].history['val_accuracy']))\n            )\n            \n            \n    def plot_accuracy_chart(self, accuracy='val_accuracy'):\n        # set the image size\n        plt.figure(figsize=(15,5))\n        \n        # plot the accuracy lines\n        for i in range(len(self.models)):\n            sns.lineplot(\n                x=range(self.epochs),\n                y=self.history[i].history[accuracy], \n                label=self.model_names[i]\n            )\n            \n            \n    def predict(self, model_name, _x_predict):\n        # get the model\n        given_model = self.models[self.model_names.index(model_name)]\n        \n        return given_model.predict_classes(_x_predict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Experiment 1\nChanging the number of Conv layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"# configurations of each of the model\nmodel_confs = [\n    {\"feature_maps\": [32], \"kernel_size\": 5, \"dense_size\": [256], \"drop_rate\": 0.0},\n    {\"feature_maps\": [32, 64], \"kernel_size\": 5, \"dense_size\": [256], \"drop_rate\": 0.0},\n    {\"feature_maps\": [32, 64, 128], \"kernel_size\": 5, \"dense_size\": [256], \"drop_rate\": 0.0}\n]\n\ncnn_models = CNN_Modeling(model_confs)\n\ncnn_models.train_models(X_train, y_train, X_val, y_val)\n\ncnn_models.plot_accuracy_chart()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Experiment 2\nChange the size of the Dense layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"# will be creating 8 models with following sizes\n[(2**(i+4)*(i!=0)) for i in range(8)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# configuration of each of the model\nmodel_confs = [\n    {\"feature_maps\": [32, 64, 128], \"kernel_size\": 5, \"dense_size\": [2**(i+4)*(i!=0)], \"drop_rate\": 0.0} for i in range(8)\n]\n\nfor each in model_confs:\n    print(each)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_models = CNN_Modeling(model_confs)\n\ncnn_models.train_models(X_train, y_train, X_val, y_val)\n\ncnn_models.plot_accuracy_chart()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Experiment 3\nChaning different drop out rates"},{"metadata":{"trusted":true},"cell_type":"code","source":"# will be creating 8 models with following dropouts\nprint([round(0.1*i,2) for i in range(8)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# configuration of each of the model\nmodel_confs = [\n    {\"feature_maps\": [32, 64, 128], \"kernel_size\": 5, \"dense_size\": [256], \"drop_rate\": round(0.1*i,2)} for i in range(8)\n]\n\nfor each in model_confs:\n    print(each)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_models = CNN_Modeling(model_confs)\ncnn_models.train_models(X_train, y_train, X_val, y_val)\ncnn_models.plot_accuracy_chart()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_df = pd.read_csv(data_dir+\"test.csv\")\npredict_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the labels of the data\nimg_ids = predict_df.values[:, 0]\n\n# extract the pixel values from the dataframe\nX_predict = predict_df.values[:, 1:]/255.0 # all the columns but 1st\n\n# reshape each row into 28x28 size\nX_predict = X_predict.reshape(-1, 28, 28, 1) # -1 tells the system to automatically figure out the size of the first dimention\n\nprint(\"Shape of X: \", X_predict.shape)\nprint(\"Sample of X: \")\nplt.imshow(X_predict[0].reshape(28, 28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_labels = cnn_models.predict(\"32C5P1-64C5P1-128C5P1-256-D30\", X_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_prediction = pd.DataFrame()\nfinal_prediction['id'] = img_ids\nfinal_prediction['label'] = predicted_labels\nfinal_prediction.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}