{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### -- DIGIT RECOGNIZER DATASET WITH CNN USING KERAS --\n1. [IMPORT THE DATA AND A QUICK LOOK](#1)\n     * [Getting the Test Data](#2)\n     * [Creating x_train and y_train](#3)\n     * [A quick visualization](#4)\n     * [Looking for sample pictures](#5)\n     \n     \n1. [EDITS FOR MAKING THE DATA SUITABLE FOR FUTURE](#6)     \n     * [Setting the shapes](#7)\n     * [Train-Test-Split](#8)\n1. [IMPORTING LIBRARIES AND BUILDING THE MODEL](#9)\n     * [Defining the Optimizer](#10)\n     * [Compiling](#11)\n     * [Epochs and Batch Size](#12)\n     * [Data Augmentation](#13)\n1. [FITTING THE MODEL](#14)\n1. [VISUALIZATION](#15)","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns \n\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"1\"></a>\n# IMPORT THE DATA AND A QUICK LOOK","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nprint(train.shape)\ntrain.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"2\"></a>\n### Lets get our test data","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nprint(test.shape)\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"3\"></a>\n### Lets Create x_train and y_train","metadata":{}},{"cell_type":"code","source":"y_train = train[\"label\"]\nx_train = train.drop([\"label\"],axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"4\"></a>\n> ### A Quick Visualization","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (15,7))\nsns.countplot(y_train,palette = \"icefire\")\nplt.title(\"Number of digit classes\")\ny_train.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"5\"></a>\n### Lets look at what we have..","metadata":{}},{"cell_type":"code","source":"# plot some samples\nimg = x_train.iloc[0].values\nimg = img.reshape((28,28))\nplt.imshow(img,cmap='gray')\nplt.title(train.iloc[0,0])\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot some samples\nimg = x_train.iloc[51].values\nimg = img.reshape((28,28))\nplt.imshow(img,cmap='gray')\nplt.title(train.iloc[51,0])\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train / 255.0\ntest = test / 255\nprint(x_train.shape)\nprint(test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"6\"></a>\n# Lets Start Having Our Data Suitable for the Models","metadata":{}},{"cell_type":"markdown","source":"<a id = \"7\"></a>\n### We should set the shapes for implementation","metadata":{}},{"cell_type":"code","source":"x_train = x_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)\nprint(\"x_train: \",x_train.shape)\nprint(\"test: \",test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils.np_utils import to_categorical # convert to one-hot-encoding / label encoding\ny_train = to_categorical(y_train,num_classes = 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"8\"></a>\n### Train Test Split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train,x_val,y_train,y_val = train_test_split(x_train,y_train,test_size = 0.1,random_state = 2)\n\nprint(\"x_train shape: \",x_train.shape)\nprint(\"x_val shape: \",x_val.shape)\nprint(\"y_train shape: \",y_train.shape)\nprint(\"y_val shape: \",y_val.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"9\"></a>\n# Import Necessary Libraries and Create the Model","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport itertools\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Dropout,Flatten,Conv2D,MaxPool2D,BatchNormalization\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\nmodel = Sequential()\n#\nmodel.add(Conv2D(filters = 32,kernel_size = (5,5),padding = \"Same\",\n                activation = \"relu\",input_shape = (28,28,1)))\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.25))\nmodel.add(BatchNormalization(\n    axis=-1,\n    momentum=0.99,\n    epsilon=0.001,\n    center=True,\n    scale=True,\n    beta_initializer=\"zeros\",\n    gamma_initializer=\"ones\",\n    moving_mean_initializer=\"zeros\",\n    moving_variance_initializer=\"ones\"\n))\n\nmodel.add(Conv2D(filters = 64,kernel_size = (3,3),padding = \"Same\",\n                activation = \"relu\"))\nmodel.add(MaxPool2D(pool_size = (2,2), strides = (2,2)))\nmodel.add(Dropout(0.25))\nmodel.add(BatchNormalization(\n    axis=-1,\n    momentum=0.99,\n    epsilon=0.001,\n    center=True,\n    scale=True,\n    beta_initializer=\"zeros\",\n    gamma_initializer=\"ones\",\n    moving_mean_initializer=\"zeros\",\n    moving_variance_initializer=\"ones\"\n))\n\nmodel.add(Conv2D(filters = 128,kernel_size = (3,3),padding = \"Same\",\n                activation = \"relu\"))\nmodel.add(MaxPool2D(pool_size = (2,2), strides = (2,2)))\nmodel.add(Dropout(0.25))\nmodel.add(BatchNormalization(\n    axis=-1,\n    momentum=0.99,\n    epsilon=0.001,\n    center=True,\n    scale=True,\n    beta_initializer=\"zeros\",\n    gamma_initializer=\"ones\",\n    moving_mean_initializer=\"zeros\",\n    moving_variance_initializer=\"ones\"\n))\n\nmodel.add(Conv2D(filters = 256,kernel_size = (3,3),padding = \"Same\",\n                activation = \"relu\"))\nmodel.add(MaxPool2D(pool_size = (2,2), strides = (2,2)))\nmodel.add(Dropout(0.25))\nmodel.add(BatchNormalization(\n    axis=-1,\n    momentum=0.99,\n    epsilon=0.001,\n    center=True,\n    scale=True,\n    beta_initializer=\"zeros\",\n    gamma_initializer=\"ones\",\n    moving_mean_initializer=\"zeros\",\n    moving_variance_initializer=\"ones\"\n))\n\n\n# Fully Connected\nmodel.add(Flatten())\n\nmodel.add(Dense(256,activation = \"relu\")) #hidden layer\nmodel.add(BatchNormalization())\n          \nmodel.add(Dense(120,activation = \"relu\"))\nmodel.add(BatchNormalization())\n          \nmodel.add(Dense(120,activation = \"relu\"))\nmodel.add(BatchNormalization())\n          \nmodel.add(Dense(120,activation = \"relu\"))\nmodel.add(BatchNormalization())\n          \nmodel.add(Dense(100,activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(BatchNormalization())\n\nmodel.add(Dense(10,activation = \"softmax\")) #output layer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"10\"></a>\n### We will define the optimizer","metadata":{}},{"cell_type":"code","source":"optimizer = Adam(lr = 0.001,beta_1=0.9, beta_2=0.999) # the optimizer tries to find the best learning rate for our model.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"11\"></a>\n### Compile Model\n* Categorical crossenthropy for multi class","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer=optimizer,loss = \"categorical_crossentropy\",metrics = [\"accuracy\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"12\"></a>\n## Epochs and Batch Size\n* Say you have a dataset of 10 examples (or samples). You have a batch size of 2, and you've specified you want the algorithm to run for 3 epochs. Therefore, in each epoch, you have 5 batches (10/2 = 5). Each batch gets passed through the algorithm, therefore you have 5 iterations per epoch.\n* reference: https://stackoverflow.com/questions/4752626/epoch-vs-iteration-when-training-neural-networks","metadata":{}},{"cell_type":"code","source":"epochs = 60\nbatch_size = 250","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"13\"></a>\n### Data Augmentation\n * To avoid overfitting we take one pic and by changing it produce new pics from that","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(rotation_range=0.9,\n                            zoom_range= 0.5,\n                            width_shift_range= 0.9,\n                            height_shift_range=0.5)\ndatagen.fit(x_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"14\"></a>\n# Fit the model","metadata":{}},{"cell_type":"code","source":"history = model.fit_generator(datagen.flow(x_train,y_train,batch_size = batch_size),\n                             epochs=epochs,validation_data= (x_val,y_val),steps_per_epoch=x_train.shape[0] // batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Achived 98.64% Accuracy!","metadata":{}},{"cell_type":"markdown","source":"<a id = \"15\"></a>\n## Lets see what the loss looks like","metadata":{}},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation\nplt.plot(history.history[\"val_loss\"],color = \"g\",label = \"validation loss\")\nplt.title(\"Test Loss\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets visualize the confusion matrix by using SNS heatmap","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n# Predict the values from the validation dataset\ny_pred = model.predict(x_val)\n# Convert predictions classes to one hot vectors\ny_pred_classes = np.argmax(y_pred,axis = 1)\n# Convert validation observations to one hot vectors \ny_true = np.argmax(y_val,axis = 1)\n# Compute the confusion matrix\nconfusion_mtx = confusion_matrix(y_true,y_pred_classes)\n# plot the cf\nf,ax = plt.subplots(figsize = (8,8))\nsns.heatmap(confusion_mtx,annot = True,linewidths=0.01,cmap = \"Greens\",linecolor = \"gray\",fmt = \".1f\",ax = ax)\nplt.xlabel(\"Predicted Label\")\nplt.ylabel(\"True Label\")\nplt.title(\"Confusion Matrix\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}