{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2021-06-16T04:12:30.385500Z","iopub.execute_input":"2021-06-16T04:12:30.385772Z","iopub.status.idle":"2021-06-16T04:12:39.090714Z","shell.execute_reply.started":"2021-06-16T04:12:30.385726Z","shell.execute_reply":"2021-06-16T04:12:39.090018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train =  pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-06-16T04:12:42.275741Z","iopub.execute_input":"2021-06-16T04:12:42.276025Z","iopub.status.idle":"2021-06-16T04:12:47.618092Z","shell.execute_reply.started":"2021-06-16T04:12:42.275977Z","shell.execute_reply":"2021-06-16T04:12:47.617264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T04:12:50.119640Z","iopub.execute_input":"2021-06-16T04:12:50.119912Z","iopub.status.idle":"2021-06-16T04:12:50.153768Z","shell.execute_reply.started":"2021-06-16T04:12:50.119866Z","shell.execute_reply":"2021-06-16T04:12:50.153069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train.iloc[:,0]","metadata":{"execution":{"iopub.status.busy":"2021-06-16T04:12:52.022512Z","iopub.execute_input":"2021-06-16T04:12:52.022790Z","iopub.status.idle":"2021-06-16T04:12:52.027074Z","shell.execute_reply.started":"2021-06-16T04:12:52.022744Z","shell.execute_reply":"2021-06-16T04:12:52.025955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T04:12:53.394488Z","iopub.execute_input":"2021-06-16T04:12:53.394767Z","iopub.status.idle":"2021-06-16T04:12:53.454730Z","shell.execute_reply.started":"2021-06-16T04:12:53.394716Z","shell.execute_reply":"2021-06-16T04:12:53.453741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T04:13:00.442985Z","iopub.execute_input":"2021-06-16T04:13:00.443286Z","iopub.status.idle":"2021-06-16T04:13:02.922848Z","shell.execute_reply.started":"2021-06-16T04:13:00.443241Z","shell.execute_reply":"2021-06-16T04:13:02.922020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking Null Values","metadata":{}},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:32:08.374631Z","iopub.execute_input":"2021-05-23T11:32:08.374968Z","iopub.status.idle":"2021-05-23T11:32:08.420921Z","shell.execute_reply.started":"2021-05-23T11:32:08.374898Z","shell.execute_reply":"2021-05-23T11:32:08.419945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1) \nX_train = X_train / 255.0\ntest = test / 255.0","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:33:06.847423Z","iopub.execute_input":"2021-05-23T11:33:06.847723Z","iopub.status.idle":"2021-05-23T11:33:08.139009Z","shell.execute_reply.started":"2021-05-23T11:33:06.847673Z","shell.execute_reply":"2021-05-23T11:33:08.138219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x = y_train.unique(), y=y_train.value_counts())\nplt.xlabel('Digits')\nplt.ylabel('Number of image samples')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Looking at this class distribution graph, I can see that the classes are almost evenly distributed. Therefore, it does not require any resampling to be applied","metadata":{}},{"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:33:33.033142Z","iopub.execute_input":"2021-05-23T11:33:33.033442Z","iopub.status.idle":"2021-05-23T11:33:33.041066Z","shell.execute_reply.started":"2021-05-23T11:33:33.033395Z","shell.execute_reply":"2021-05-23T11:33:33.040339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We are reshaping the data into images.\n\n* -1 is given so that the number of rows are dynamically written as per the dataset\n* 28*28 is the size of the image\n* 1 is channel as Images are in Black and White (Note - If it is in RGB, then channel value should be 3)","metadata":{}},{"cell_type":"code","source":"Y_train = tf.keras.utils.to_categorical(Y_train, 10)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:33:35.582711Z","iopub.execute_input":"2021-05-23T11:33:35.583026Z","iopub.status.idle":"2021-05-23T11:33:35.58845Z","shell.execute_reply.started":"2021-05-23T11:33:35.582973Z","shell.execute_reply":"2021-05-23T11:33:35.587297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting into Train and Validation Datasets\n\nI split the dataset into train and validation datasets, and then while training, we can check accuracy by the validation dataset, which the neural network didn't see. I have chosen a 20%/80% split because it is the most common way to split the dataset.","metadata":{}},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\nnum_classes = 10\nepochs = 50","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Building","metadata":{}},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Conv2D(64, (3, 3), input_shape=(28, 28, 1), activation=tf.nn.relu),\n    tf.keras.layers.MaxPooling2D(),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Conv2D(64, (3, 3), activation=tf.nn.relu, padding=\"SAME\"),\n    tf.keras.layers.MaxPooling2D(),\n    tf.keras.layers.Dropout(0.2),\n    \n    tf.keras.layers.Conv2D(128, (3, 3), activation=tf.nn.relu, padding=\"SAME\"),\n    tf.keras.layers.MaxPooling2D(),\n    tf.keras.layers.Dropout(0.2),\n\n    tf.keras.layers.Conv2D(128, (3, 3), activation=tf.nn.relu, padding=\"SAME\"),\n    tf.keras.layers.MaxPooling2D(),\n    tf.keras.layers.BatchNormalization(),\n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dense(10, activation=\"softmax\")\n])\n\nmodel.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The Input Layer is defined to read the images into the CNN\n* Convolution Layer - These are important when nearby associations among the features matter. Here it is less likely that the pixels in opposite corner of the images will contribute to the classification. Therefore, I used the convolutional layer\n* Max Pooling Layer - It selects the maximum element from the region of the feature map covered by the filter. So, the output would be a feature map containing the most prominent features of the previous feature map.\n* Batch Normalization - It is technique for training very deep neural networks that standardizes the inputs to a layer for each mini-batch. This has the effect of stabilizing the learning process and dramatically reducing the number of training epochs required to train deep networks.\n* Dropout - It is a way of cutting too much association among features by dropping a specific percentage of weights randomly.\n* Flatten - It is used when we get multidimensional output and want to make it linear to pass it onto a Dense layer. It is equivalent to numpy.ravel (Look at the shape before and flatten in the model summary)\n* Dense - are used when association can exist among any feature to any other feature in data point\n\n**The output layer in a classification problem should always be a softmax layer with the number of output classes as its paramter**\n","metadata":{}},{"cell_type":"markdown","source":"# Image Augmentation\nTo improve model's probability power, I will use Image Augmentations.","metadata":{}},{"cell_type":"code","source":"image_data_gen = tf.keras.preprocessing.image.ImageDataGenerator(\n        featurewise_center=False,\n        samplewise_center=False,\n        featurewise_std_normalization=False,\n        samplewise_std_normalization=False,\n        zca_whitening=False,\n        rotation_range=10, \n        zoom_range = 0.1,\n        width_shift_range=0.1,\n        height_shift_range=0.1,\n        horizontal_flip=False,\n        vertical_flip=False)\n\nimage_data_gen.fit(X_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Convolutional Neural Network","metadata":{}},{"cell_type":"code","source":"h = model.fit_generator(image_data_gen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_loss, final_acc = model.evaluate(X_val, Y_val, verbose=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting results","metadata":{}},{"cell_type":"code","source":"train_accuracies = h.history[\"accuracy\"]\nvalidation_accuracies = h.history[\"val_accuracy\"]\n\nepochs_range = list(range(1, epochs+1))\n\nscatter_accuracy_hovertemplate = \"Epoch: %{x}<br>Accuracy: %{y}\"\n\ndata = [\n    go.Scatter(x=epochs_range, y=train_accuracies, name=\"Train Accuracy\", hovertemplate=scatter_accuracy_hovertemplate),\n    go.Scatter(x=epochs_range, y=validation_accuracies, name=\"Validation Accuracy\", hovertemplate=scatter_accuracy_hovertemplate)\n]\n\ntitle = go.layout.Title(text=\"Train & Validation Accuracies\",\n                        font=dict(size=25))\n\nlayout = go.Layout(template=\"plotly_dark\",\n                    xaxis=dict(ticks=\"outside\", ticklen=5, tickvals=list(range(0, epochs+1, 2)), tickfont=dict(size=10), title=dict(text=\"Epochs\", font=dict(size=17))),\n                    yaxis=dict(title=dict(text=\"Accuracy\", font=dict(size=14))),\n                    title=title)\n\nfigure = go.Figure(data=data, layout=layout)\nfigure.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_losses = h.history[\"loss\"]\nvalidation_losses = h.history[\"val_loss\"]\n\nscatter_loss_hovertemplate = \"Epoch: %{x}<br>Loss: %{y}\"\n\ndata = [\n    go.Scatter(x=epochs_range, y=train_losses, name=\"Train Loss\", hovertemplate=scatter_loss_hovertemplate),\n    go.Scatter(x=epochs_range, y=validation_losses, name=\"Validation Loss\", hovertemplate=scatter_loss_hovertemplate)\n]\n\ntitle = go.layout.Title(text=\"Train & Validation Losses\",\n                        font=dict(size=25))\n\nlayout = go.Layout(template=\"plotly_dark\",\n                    xaxis=dict(ticks=\"outside\", ticklen=5, tickvals=list(range(0, epochs+1, 2)), tickfont=dict(size=10), title=dict(text=\"Epochs\", font=dict(size=17))),\n                    yaxis=dict(title=dict(text=\"Loss\", font=dict(size=14))),\n                    title=title)\n\nfigure = go.Figure(data=data, layout=layout)\nfigure.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = model.predict(X_val)\nY_pred_classes = np.argmax(Y_pred,axis = 1) \nY_true = np.argmax(Y_val,axis = 1) \nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_mtx","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results=model.predict(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"result.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}