{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport seaborn as sb\nfrom sklearn.linear_model import LogisticRegression \nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import svm\nfrom sklearn import metrics\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"id":"fQpnxT4QgFwO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntrain.shape","metadata":{"id":"0D5D6vk8heuB","outputId":"b2b4d478-0240-47a1-9415-e54e0b67623b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"id":"JXC5i0bQhrPL","outputId":"9773e7fc-977c-4ed4-b85a-83d67f42904d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Let's check how many examples do we have for each of the digit.","metadata":{"id":"JKBpUDSohyQX"}},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nsb.countplot(train['label'])\nplt.show()","metadata":{"id":"zAk60Fwghtu4","outputId":"0752999a-0920-49eb-ae1e-981a79b36d17"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing some of the examples from the given dataset.","metadata":{"id":"WbWY1nrAw3XR"}},{"cell_type":"code","source":"plt.subplots(figsize=(15,5))\n\nfor i in range(24):\n  plt.subplot(3,8,i+1)\n  k = np.random.randint(10000)\n  plt.imshow(train.loc[k][1:].values.reshape(28,28))\n  plt.title('Label ' + str(train.loc[k][0:1][0]))\nplt.tight_layout()\nplt.show()","metadata":{"id":"t-IUlq0ww2ys","outputId":"209cb9da-1991-4ec7-9b32-6baa544ec614"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline\nLet's check what is the base line solution for the test data.","metadata":{"id":"86hxQhJxiQNE"}},{"cell_type":"code","source":"ss = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nss['Label'] = 4\nss.to_csv('Submission.csv',index=False)\nss.head()","metadata":{"id":"jAQ0VtpAiCZ0","outputId":"2f2d4409-8c60-4938-90d8-468eb2e915db"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Hence, as expected we get the 10% accuracy on the categorization of the digits based on the pixel value.\n\n* So, 10% is our baseline accuracy.","metadata":{"id":"hsobz7hqi8In"}},{"cell_type":"markdown","source":"## Logistic Regression And SVM\n\n* First let's try to predict the digits using a logistic regression and an svm model.\n\n## Neural Network\n* Secondly we will try to build Neural Network classifier to learn complex non-linear function which can fit our function even better without overfitting in keras.","metadata":{"id":"9PMYXn1ujTh4"}},{"cell_type":"code","source":"features = train.drop(['label'],axis=1)\ntarget = train['label']\n\nX_train, X_val, Y_train, Y_val = train_test_split(features, target, test_size=0.05, random_state=10)\n\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_val = scaler.transform(X_val)\n","metadata":{"id":"sZrGSmVbmW64"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_log = LogisticRegression()\nmodel_log.fit(X_train, Y_train)\nprint('Logistic Regression:')\nY_pred = model_log.predict(X_train)\nprint('Training Accuracy : ',metrics.accuracy_score(Y_train, Y_pred))\n\nY_pred = model_log.predict(X_val)\nprint('Validation Accuracy : ',metrics.accuracy_score(Y_val, Y_pred))","metadata":{"id":"R6CyxY0Fidm8","outputId":"ee268098-26cd-4f1e-8ccf-7e2c7719f07e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_svm = svm.SVC(kernel=\"linear\")\n\nmodel_svm.fit(X_train, Y_train)\nprint('Support Vector Machine:')\nY_pred = model_svm.predict(X_train)\nprint('Training Accuracy : ',metrics.accuracy_score(Y_train, Y_pred))\n\nY_pred = model_svm.predict(X_val)\nprint('Validation Accuracy : ',metrics.accuracy_score(Y_val, Y_pred))\nprint()","metadata":{"id":"etDGH-sEnHt7","outputId":"105e22df-e84b-4ad8-f342-bdaaadeeb2f7"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* #### We are able to get an acuuracy rate of 91% using Logistic regression model.\n* #### SVM gives an accuracy rate of 92% which is an improvement from Logistic regression but SVM is overfitting on the training data we can observe from the training and the validation accuracy difference.","metadata":{"id":"9GYUMJ2zpAqJ"}},{"cell_type":"markdown","source":"### Neural Network: For digit classification.","metadata":{"id":"uZGbrCU1xAjC"}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom keras import layers\n\nmodel_nn = keras.Sequential([\n      layers.Dense(256, activation='relu',input_shape=[784]),\n      layers.BatchNormalization(),\n      layers.Dense(256, activation='relu'),\n      layers.Dropout(0.3),\n      layers.BatchNormalization(),\n      layers.Dense(10, activation='softmax')\n])\n\nmodel_nn.compile(\n    loss='sparse_categorical_crossentropy',\n    optimizer='adam',\n    metrics=['sparse_categorical_accuracy']\n)","metadata":{"id":"bnpYL6A7siS0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_nn.summary()","metadata":{"id":"e-KiJW9F5Ymy","outputId":"a052b615-d39e-4d26-f3bd-d93351ef8377"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_nn.fit(X_train, Y_train,\n                    epochs = 5,\n                    validation_data = (X_val, Y_val))","metadata":{"id":"8xWIdZrozU4m","outputId":"ae258ff0-d32f-4130-ee34-86845e59d8d3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df.loc[:,['loss','val_loss']].plot()\nhistory_df.loc[:,['sparse_categorical_accuracy','val_sparse_categorical_accuracy']].plot()\nplt.show()","metadata":{"id":"YDzcwIlzzV9c","outputId":"10e2d65d-b83e-483e-edf5-ac10633e1f97"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### So, here we can observe that the neural network classify the digits with an accuracy rate of 98% and it isn't overfitting the training data which is visible from the learning curves.","metadata":{"id":"ZLMD_tImzJLn"}},{"cell_type":"code","source":"test = pd.read_csv('../input/digit-recognizer/test.csv')\ntest = scaler.transform(test)","metadata":{"id":"NRzevLxj0rFr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss['Label'] = np.argmax(model_nn.predict(test), axis=1)\nss.to_csv('Submission_NN.csv',index=False)\nss.head()","metadata":{"id":"C49eImlb5BZq","outputId":"d4935b3c-2aed-491e-be6f-0a19b29fb118"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Why not to try Convolutional Neural network as well?","metadata":{"id":"TFfHuRu7KOLl"}},{"cell_type":"code","source":"X_train = X_train.reshape((39900,28,28,1))\nX_val = X_val.reshape((2100,28,28,1))","metadata":{"id":"wR5iulTqKmLk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cnn = tf.keras.models.Sequential([\n                                    layers.Conv2D(16,(3,3),activation='relu',input_shape=(28,28,1),padding='same'),\n                                    layers.Conv2D(32,(3,3),activation='relu',padding='same'),\n                                    layers.Conv2D(64,(3,3),activation='relu',padding='same'),\n                                    layers.MaxPooling2D(2,2),\n                                    layers.Conv2D(128,(3,3),activation='relu',padding='same'),\n                                    \n\n                                    layers.Flatten(),\n                                    layers.Dense(512,activation='relu'),\n                                    layers.BatchNormalization(),\n                                    layers.Dense(256,activation='relu'),\n                                    layers.Dropout(0.3),\n                                    layers.BatchNormalization(),\n                                    layers.Dense(10,activation='softmax')\n])","metadata":{"id":"T-xOGN8JJ6z3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cnn.summary()","metadata":{"id":"gCmWHhNCK7Y4","outputId":"fc7f59ea-d2cc-4738-c21e-3dd2fc5341dc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cnn.compile(\n    loss='sparse_categorical_crossentropy',\n    optimizer='adam',\n    metrics=['sparse_categorical_accuracy']\n)","metadata":{"id":"8sDzjYsXLBsX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = model_cnn.fit(X_train, Y_train,\n                 epochs=5,\n                 validation_data=(X_val.reshape(2100,28,28,1),Y_val)\n                 )","metadata":{"id":"yHzPeD_HLKA6","outputId":"6d8a83b3-972d-45fc-dc5b-9454dd853ab1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df.loc[:,['loss','val_loss']].plot()\nhistory_df.loc[:,['sparse_categorical_accuracy','val_sparse_categorical_accuracy']].plot()\nplt.show()","metadata":{"id":"6cDp2wV7LaMk","outputId":"7d43ad50-1a22-40af-8c44-a2eb062a91cc"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### So, this is the power of CNN accuracy has been shot up to 99% without overfitting the data.","metadata":{"id":"IA5Ir0K3MFLv"}},{"cell_type":"code","source":"ss['Label'] = np.argmax(model_cnn.predict(test.reshape((28000,28,28,1))), axis=1)\nss.to_csv('Submission_CNN.csv',index=False)\nss.head()","metadata":{"id":"E2aCihNzMEAu","outputId":"07818943-4cdc-455f-8aad-d860a63ff96a"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizing convolutions","metadata":{"id":"ZCWowxbBPriF"}},{"cell_type":"code","source":"layer_outputs = [layer.output for layer in model_cnn.layers]\nactivation_model = keras.models.Model(inputs = model_cnn.input, outputs=layer_outputs)","metadata":{"id":"n1BxbLaXPwCd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"digits_index = [np.random.randint(10000) for i in range(5)]\n\nfor i in digits_index:\n  plt.subplots(figsize=(15,15))\n  plt.subplot(1,8,1)\n  plt.imshow(X_train[i].reshape((28,28)))\n  f1 = activation_model.predict(X_train[i].reshape(1,28,28,1))\n\n  for j in range(2):\n    plt.subplot(1,7,j*3 + 2)\n    plt.imshow(f1[1][0, :, :, j*4+5])\n    plt.subplot(1,7,j*3 + 3)\n    plt.imshow(f1[2][0, :, :, j*4+5])\n    plt.subplot(1,7,j*3 + 4)\n    plt.imshow(f1[3][0, :, :, j*4+5])\n  plt.tight_layout()\n  plt.show()","metadata":{"id":"Gb5NmrxfQQ20","outputId":"ccddb7f9-c328-4e15-cbd5-bd10f6b42c64"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observe that how the convolutions have emphasized arcs, lines which are working as a feature to learn a better model for our task to digit classification.","metadata":{"id":"2Ji9AwiIXhGN"}}]}