{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\n\nsns.set(style='white', context='notebook', palette='deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train['label']\nX_train = train.drop('label', axis=1)\n\ndel train #To free more space","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y_train)\ny_train.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Similarly distributed for each digit."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Corrupted images in train: ',X_train.isnull().any().any())\nprint('Corrupted images in test: ',test.isnull().any().any())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train / 255.0\ntest = test / 255.0\n\n#Normalising of the data to a [0 - 1] range. CNN converges faster than on [0 - 255]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)\n\n#Reshaping to 3D, the images are 28 x 28 x 1. Seeing as they are black and white, they only use 1 channel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# One hot encoding. For example, the value one would equal [0,1,0,0,0,0,0,0,0,0]\n\ny_train = to_categorical(y_train, num_classes = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=42)\n\n#The distribtion of y labels is fairly equal as seen in the graph earlier. Therefore, a random split of 10% validation and 90% training won't cause\n#certain labels to be over represented.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(X_train[42][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Basic model\n# [Conv2D -> Relu -> Conv2D -> Relu -> MaxPool2D -> Dropout -> Flatten -> Dense -> Dropout -> Out]\n\nmodel1 = Sequential()\n\nmodel1.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))\n\nmodel1.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel1.add(MaxPool2D(pool_size=(2, 2)))\n\nmodel1.add(Dropout(0.25))\n\nmodel1.add(Flatten())\nmodel1.add(Dense(128, activation='relu'))\nmodel1.add(Dropout(0.5))\nmodel1.add(Dense(10, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.compile(optimizer = 'adam' , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = model1.fit(X_train, y_train,\n          batch_size=128, epochs=30,\n          verbose=2,\n          validation_data=(X_val, y_val))\n\nprint('\\n')\nprint('Best training score: ', max(history1.history['accuracy']))\nprint('Best validation score: ', max(history1.history['val_accuracy']))\n\n# Plotting the results\nfig = plt.figure()\nplt.subplot(2,1,1)\nplt.plot(history1.history['accuracy'])\nplt.plot(history1.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='lower right')\n\nplt.subplot(2,1,2)\nplt.plot(history1.history['loss'])\nplt.plot(history1.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper right')\n\nplt.tight_layout()\n\nfig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        rotation_range=10,  \n        zoom_range = 0.10,  \n        width_shift_range=0.1, \n        height_shift_range=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1_aug = model1.fit_generator(datagen.flow(X_train, y_train,\n          batch_size=128), epochs=30,\n          verbose=2,\n          validation_data=(X_val, y_val))\n\nprint('\\n')\nprint('Best training score: ', max(history1_aug.history['accuracy']))\nprint('Best validation score: ', max(history1_aug.history['val_accuracy']))\n\n# Plotting the results\nfig = plt.figure()\nplt.subplot(2,1,1)\nplt.plot(history1_aug.history['accuracy'])\nplt.plot(history1_aug.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='lower right')\n\nplt.subplot(2,1,2)\nplt.plot(history1_aug.history['loss'])\nplt.plot(history1_aug.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper right')\n\nplt.tight_layout()\n\nfig","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model2"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set the CNN model \n# my CNN architechture is In -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*3 -> Flatten -> Dense -> Dropout -> Out\n#Credit to \"https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\"\n\nmodel2 = Sequential()\n\nmodel2.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu', input_shape = (28,28,1)))\nmodel2.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu'))\nmodel2.add(MaxPool2D(pool_size=(2,2)))\nmodel2.add(Dropout(0.25))\n\n\nmodel2.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu'))\nmodel2.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu'))\nmodel2.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel2.add(Dropout(0.25))\n\nmodel2.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu'))\nmodel2.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu'))\nmodel2.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel2.add(Dropout(0.25))\n\n\nmodel2.add(Flatten())\nmodel2.add(Dense(256, activation = \"relu\"))\nmodel2.add(Dropout(0.5))\nmodel2.add(Dense(10, activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.compile(optimizer = 'RMSprop' , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history2 = model2.fit(X_train, y_train,\n          batch_size=128, epochs=30,\n          verbose=2,\n          validation_data=(X_val, y_val))\n\nprint('\\n')\nprint('Best training score: ', max(history2.history['accuracy']))\nprint('Best validation score: ', max(history2.history['val_accuracy']))\n\n# plotting the metrics\nfig = plt.figure()\nplt.subplot(2,1,1)\nplt.plot(history2.history['accuracy'])\nplt.plot(history2.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='lower right')\n\nplt.subplot(2,1,2)\nplt.plot(history2.history['loss'])\nplt.plot(history2.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper right')\n\nplt.tight_layout()\n\nfig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history2_aug = model2.fit_generator(datagen.flow(X_train, y_train,\n          batch_size=128), epochs=30,\n          verbose=2,\n          validation_data=(X_val, y_val))\n\nprint('\\n')\nprint('Best training score: ', max(history2_aug.history['accuracy']))\nprint('Best validation score: ', max(history2_aug.history['val_accuracy']))\n\n# plotting the metrics\nfig = plt.figure()\nplt.subplot(2,1,1)\nplt.plot(history2_aug.history['accuracy'])\nplt.plot(history2_aug.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='lower right')\n\nplt.subplot(2,1,2)\nplt.plot(history2_aug.history['loss'])\nplt.plot(history2_aug.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper right')\n\nplt.tight_layout()\n\nfig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Comparing the models with and without data augmentation\n\nprint('Model 1: Best Train accuracy={0:.5f}, Best Validation accuracy={1:.5f}'.format( max(history1.history['accuracy']), max(history1.history['val_accuracy']) ) )\nprint('Model 2: Best Train accuracy={0:.5f}, Best Validation accuracy={1:.5f}'.format(max(history2.history['accuracy']), max(history2.history['val_accuracy'])))\n\nprint('\\nModel 1 with data augmentation: Best Train accuracy={0:.5f}, Best Validation accuracy={1:.5f}'.format(max(history1_aug.history['accuracy']), \n                                                                                                             max(history1_aug.history['val_accuracy'])))\n\nprint('Model 2: with data augmentation: Best Train accuracy={0:.5f}, Best Validation accuracy={1:.5f}'.format(max(history2_aug.history['accuracy']), \n                                                                                                              max(history2_aug.history['val_accuracy'])))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = model1.predict(test)\n\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")\n\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"MNIST-CNN-NoAug-RMS.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}