{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NOTE  \n### 7 models. Starting with a basic model and progressing to more deeper models. \n### Did not consider filtering out the models which failed to give best result as a depiction of the learning process, this being my first CNN project. \n### First 2 models are rather shallow and don't capture the intricacies of the image well Next is a deeper model with BatchNormalisation and Dropout. Using transferlearning, LeNet50 and ResNet50 (with imagenet weights) models are implemented taking on-the-fly augmented data, some training only selected layers at the end of the model. \n### Lastly, a deep CNN model taking in augmented data and working with decaying learning rate which gave the best result.\n### The epochs for the first 6 models are **very low as they were not the best performing models**. Hence their graphs do not really do justice to the actual performance due to less plotting points.\n\n## To skip to the best performing model jump to **Cell 44**.\n\n#### If this helps you in learning, an upvote would be huge! ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cv2\nfrom PIL import Image\nimport glob\nimport keras as k\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.optimizers import SGD\nfrom keras.layers import Flatten\nfrom keras.layers import Dense \nfrom keras.layers import Conv2D\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import AveragePooling2D\nfrom keras.optimizers import Adam\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Dropout \nfrom keras.layers import Activation\nimport keras\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.resnet50 import ResNet50\nfrom numpy import loadtxt\nfrom keras.models import Model\nfrom keras.callbacks import ReduceLROnPlateau\nimport tensorflow as tf","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain = train['label']\nxtrain = train.drop(labels = [\"label\"],axis = 1) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain = xtrain / 255.0\ntest = test / 255.0\n\nxtrain = xtrain.values.reshape(-1,28,28,1)                         #Reshaping into size(examples, height, width, channel)\nxtest = test.values.reshape(-1,28,28,1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain = np.asarray(tf.one_hot(ytrain, 10, axis = -1))       #One-Hot encoding","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xtrain.shape)\nprint(ytrain.shape)\nprint(xtest.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(xtrain, ytrain, test_size=0.1, random_state=42)\nprint(x_train.shape)\nprint(x_val.shape)\nprint(y_train.shape)\nprint(y_val.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We will use the below defined function after making prediction from the trained model, to create the final submission file and visualising the model performance.","metadata":{}},{"cell_type":"code","source":"def submission_and_visualization(final_predictions, model_number, model_history):\n    y = final_predictions.copy()\n    y= np.argmax(y, axis = 1)\n    y.reshape(28000,1);\n\n    col1 = np.arange(42000,70000,1)\n    col2 = y.copy()\n\n    final = np.stack((col1, col2), axis = 1)\n    finaldf = pd.DataFrame(data=final)\n    finaldf.rename(columns = {0:'filename', 1:'label'} , inplace = True)\n\n    name = []\n    for i in finaldf['filename']:\n        i = str(i)+\".png\"\n        name.append(i)\n\n    finaldf['filenames'] = name\n    finaldf['filename']=finaldf['filenames']\n    del finaldf['filenames']\n    \n    submission_file = \"Solution\"+str(model_number)+\".csv\"\n    finaldf.to_csv(submission_file, index=False)\n\n    plt.plot(model_history.history['loss'], label='Train loss')\n    plt.plot(model_history.history['val_loss'], label='Validation loss')\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Loss\")\n    plt.title(\"Loss Graph\")\n    plt.legend()\n    plt.show()\n    plt.plot(model_history.history['accuracy'], label = 'Train accuracy')\n    plt.plot(model_history.history['val_accuracy'], label = 'Validation accuracy')\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Accuracy\")\n    plt.title(\"Accuracy Graph\")\n    plt.legend()\n    plt.show()\n    \n    return ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN Models :","metadata":{}},{"cell_type":"markdown","source":"## Model 1 \n### Basic model without data augmentation, dropout, batch normalisation etc. ","metadata":{}},{"cell_type":"code","source":"model1 = Sequential()\nmodel1.add(Conv2D(16, (3,3), activation = 'relu', kernel_initializer = 'he_uniform', input_shape = (28,28,1)))\nmodel1.add(MaxPooling2D((2,2)))\nmodel1.add(Conv2D(32, (3,3), activation = 'relu', kernel_initializer = 'he_uniform'))\nmodel1.add(MaxPooling2D((2,2)))\nmodel1.add(Flatten())\nmodel1.add(Dense(100, activation = 'relu', kernel_initializer = 'he_uniform'))\nmodel1.add(Dense(64, activation = 'relu', kernel_initializer = 'he_uniform'))\nmodel1.add(Dense(10, activation = 'softmax'))\nmodel1.compile(optimizer = Adam(), loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history1 = []                  #Using callback to store history of accuracies and losses through and with which model proggresses. \nhistory1 = model1.fit(x_train, y_train, batch_size=32, epochs = 10, validation_data=(x_val, y_val),  callbacks=[history1]) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest1 = model1.predict(xtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation of train and developer sets' performance on model1 for each epoch.","metadata":{}},{"cell_type":"code","source":"submission_and_visualization(ytest1, 1, history1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 2\n### Deeper model with Batch Normalisation. ","metadata":{}},{"cell_type":"code","source":"model2 = Sequential()\nmodel2.add(Conv2D(16, (3,3), activation = 'relu', kernel_initializer = 'he_uniform', input_shape = (28,28,1)))\nmodel2.add(MaxPooling2D((2,2)))\nmodel2.add(BatchNormalization())\nmodel2.add(Conv2D(32, (3,3), activation = 'relu', kernel_initializer = 'he_uniform'))\nmodel2.add(Conv2D(64, (3,3), activation = 'relu', padding = 'same' , kernel_initializer = 'he_uniform'))\nmodel2.add(BatchNormalization())\nmodel2.add(MaxPooling2D((2,2)))\nmodel2.add(Flatten())\nmodel2.add(Dense(100, activation = 'relu', kernel_initializer = 'he_uniform'))\nmodel2.add(Dense(64, activation = 'relu', kernel_initializer = 'he_uniform'))\nmodel2.add(Dense(10, activation = 'softmax'))\nopt = Adam(learning_rate= 0.001)\nmodel2.compile(optimizer = opt, loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2 = []\nhistory2 = model2.fit(x_train, y_train, batch_size=32, epochs = 10, validation_data=(x_val, y_val),  callbacks=[history2])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest2 = model2.predict(xtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation of train and developer sets' performance on model2 for each epoch.","metadata":{}},{"cell_type":"code","source":"submission_and_visualization(ytest2, 2, history2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 3 :\n### Deeper model with Batch Normalisation, Dropout and Adam optimizer.\n### Since this was the most promising, it was trained for 50 epochs.","metadata":{}},{"cell_type":"code","source":"model3 = Sequential()\nmodel3.add(Conv2D(32, (3, 3), kernel_initializer='he_uniform', input_shape=(28, 28, 1)))\nmodel3.add(MaxPooling2D((2, 2)))\nmodel3.add(Activation('relu'))\nmodel3.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform'))\nmodel3.add(Conv2D(64, (3, 3), kernel_initializer='he_uniform'))\nmodel3.add(MaxPooling2D((2, 2)))\nmodel3.add(Activation('relu'))\nmodel3.add(BatchNormalization())\nmodel3.add(Flatten())\nmodel3.add(Dense(100, activation='relu', kernel_initializer='he_uniform'))\nmodel3.add(BatchNormalization())\nmodel3.add(Dense(10, activation='softmax'))\nmodel3.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history3 = []\nhistory3 = model3.fit(x_train, y_train, batch_size = 32, epochs = 5, validation_data=(x_val, y_val),  callbacks=[history3])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest3 = model3.predict(xtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation of train and developer sets' performance on model3 for each epoch.","metadata":{}},{"cell_type":"code","source":"submission_and_visualization(ytest3, 3, history3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LeNet50\n### Model using LeNet50 architecture, without any changes.","metadata":{}},{"cell_type":"code","source":"model4 = Sequential()\nmodel4.add(Conv2D(6, (5,5), activation = 'tanh', input_shape = (28,28,1), padding = 'same'))\nmodel4.add(AveragePooling2D())\nmodel4.add(Conv2D(16, (5,5), activation = 'tanh', padding = 'valid'))\nmodel4.add(AveragePooling2D())\nmodel4.add(Flatten())\nmodel4.add(Dense(120, activation = 'tanh'))\nmodel4.add(Dense(84, activation = 'tanh'))\nmodel4.add(Dense(10, activation = 'sigmoid'))\nmodel4.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics=['accuracy'] )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history4 = []\nhistory4 = model4.fit(x_train, y_train, batch_size = 32, epochs = 10, validation_data=(x_val, y_val),  callbacks=[history3])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest4 = model4.predict(xtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation of train and developer sets' performance on model4 for each epoch.","metadata":{}},{"cell_type":"code","source":"submission_and_visualization(ytest4, 4, history4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transfer learning using ResNet50","metadata":{}},{"cell_type":"markdown","source":"### To use ResNet50 we need to do some preprocessing on the datasets again because\n### 1. The network can take the input image having height and width as multiples of 32.\n### 2. The network takes 3 as channel width.","metadata":{}},{"cell_type":"code","source":"train_examples = xtrain.shape[0]        #Number of training examples\ntest_examples = xtest.shape[0]          #Number of test examples \n\nResTrain = np.zeros((train_examples, 32,32,3))          \nResTest = np.zeros((test_examples, 32,32,3))\n\nfor example in range(train_examples):\n    ResTrain[example,:28,:28,0] = xtrain[example, :].reshape(28,28)\n    ResTrain[example,:28,:28,1] = xtrain[example, :].reshape(28,28)\n    ResTrain[example,:28,:28,2] = xtrain[example, :].reshape(28,28)\n\nfor example in range(test_examples):\n    ResTest[example,:28,:28,0] = xtest[example, :].reshape(28,28)\n    ResTest[example,:28,:28,1] = xtest[example, :].reshape(28,28)\n    ResTest[example,:28,:28,2] = xtest[example, :].reshape(28,28)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for example in range(train_examples):\n    ResTrain[example] = cv2.resize(ResTrain[example], (32, 32))\n    \nfor example in range(test_examples):\n    ResTest[example] = cv2.resize(ResTest[example], (32, 32))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of train input images : \", ResTrain.shape)\nprint(\"Shape of test input images : \", ResTest.shape)\nprint(\"Shape of train labels : \", ytrain.shape)\nprint(\"We have processed the input data to be fed into ResNet50.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation :","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(width_shift_range=.3, height_shift_range=.2,\n                                   shear_range=.3, rotation_range=25, zoom_range=.2)\n                                   \nvalidation_datagen = ImageDataGenerator()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading ResNet50 model with weights trained on ImageNet data, as a base model.","metadata":{}},{"cell_type":"code","source":"base_model = ResNet50(weights='imagenet', input_shape=(32,32,3),include_top=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 5 is a model built on top of ResNet50, **where all the layers are trained.** ","metadata":{}},{"cell_type":"code","source":"#Appeding layers to the base model(ResNet50)\nmodel5 = Sequential()\nmodel5.add(base_model)\nmodel5.add(Flatten())\nmodel5.add(Dense(units=60, activation='relu'))\nmodel5.add(Dense(units=10, activation='softmax'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model5.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model5.compile(optimizer='adam', loss= 'categorical_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images, val_images, train_labels, val_labels = train_test_split(ResTrain, ytrain, test_size=0.1, shuffle=True, random_state=42)\ntest_images = ResTest","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class myCallback(keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs={}):\n        if (logs.get('accuracy') > 0.999999):\n            print(\"Stop training!\")\n            self.model.stop_training = True\ncallbacks = myCallback()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 212\nhistory5 = model5.fit(train_datagen.flow(train_images,train_labels, batch_size=batch_size),\n                         steps_per_epoch=train_images.shape[0] / batch_size, \n                         epochs=10,    \n                         validation_data=validation_datagen.flow(val_images,val_labels,\n                                                                 batch_size=batch_size),\n                         validation_steps=val_images.shape[0] / batch_size,\n                         callbacks=[callbacks])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest5 = model5.predict(ResTest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation of train and developer sets' performance on model5 for each epoch.","metadata":{}},{"cell_type":"code","source":"submission_and_visualization(ytest5, 5, history5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 6 is a model built on top of ResNet50, where selected layers, at the end of the network architecture are trained. ","metadata":{}},{"cell_type":"code","source":"base_model = ResNet50(include_top=False, weights='imagenet', input_shape=(32,32,3))\nx = base_model.output\nx = Flatten()(x)\nx = Dense(units=60, activation='relu')(x)\npred = Dense(units=10, activation='softmax')(x)\n\nmodel6 = Model(inputs=base_model.input, outputs=pred)\n\nmodel6.compile(optimizer='rmsprop', loss= 'categorical_crossentropy', metrics=['accuracy'])\n\nfor layer in base_model.layers:\n    layer.trainable = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history6 = []\nhistory6 = model6.fit(train_datagen.flow(train_images,train_labels, batch_size=batch_size),\n                         steps_per_epoch=train_images.shape[0] / batch_size, \n                         epochs=10,   \n                         validation_data=validation_datagen.flow(val_images,val_labels,\n                                                                 batch_size=batch_size),\n                         validation_steps=val_images.shape[0] / batch_size,\n                         callbacks=[history6])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in base_model.layers[0:143]:\n    layer.trainable = False\n    \nfor layer in base_model.layers[143:]:\n    layer.trainable = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history6 = model6.fit(train_datagen.flow(train_images,train_labels, batch_size=batch_size),\n                         steps_per_epoch=train_images.shape[0] / batch_size, \n                         epochs=5,    \n                         validation_data=validation_datagen.flow(val_images,val_labels,\n                                                                 batch_size=batch_size),\n                         validation_steps=val_images.shape[0] / batch_size,\n                         callbacks=[history6]) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest6 = model6.predict(ResTest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualisation of train and developer sets' performance on model6 for each epoch.","metadata":{}},{"cell_type":"code","source":"submission_and_visualization(ytest6, 6, history6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model 7 :","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, (4, 4), activation = 'relu', input_shape = (28, 28, 1)))\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization(axis = -1))\n      \nmodel.add(Conv2D(128, (3, 3)))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization(axis = -1))\nmodel.add(Dropout(0.2))\n       \nmodel.add(Conv2D(128, (2, 2), activation = 'relu'))\nmodel.add(Conv2D(256, (2, 2)))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization(axis = -1))\nmodel.add(Dropout(0.2))\n        \nmodel.add(Flatten())\nmodel.add(Dense(256, activation = 'relu'))\nmodel.add(Dense(128, activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(10, activation = 'softmax'))\n\nopt = Adam(lr = 0.002)\nmodel.compile(optimizer = opt, loss = 'categorical_crossentropy',  metrics = ['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gen = ImageDataGenerator(rotation_range=.15, horizontal_flip=False, vertical_flip=False, \n                              shear_range=.1, width_shift_range=.1, height_shift_range=.1, zoom_range=.1)\nval_gen = ImageDataGenerator()\n\ntrain_img_gen = train_gen.flow(x_train, y_train, batch_size=32)\nval_img_gen = val_gen.flow(x_val, y_val, batch_size=32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_lr = ReduceLROnPlateau(monitor = 'val_accuracy', patience = 3, verbose = 1, factor = 0.4, min_lr = 0.00002,\n                                            mode = 'auto', cooldown = 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_img_gen, epochs = 80, validation_data=val_img_gen, verbose=1, callbacks=[reduce_lr])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest = model.predict(xtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_and_visualization(ytest, None, history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Index with the maximum probability to 1\nresults = np.argmax(ytest,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"submission.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}