{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NOTE  \n### 7 models. Starting with a basic model and progressing to more deeper models. \n### Did not consider filtering out the models which failed to give best result as a depiction of the learning process, this being my first CNN project. \n### First 2 models are rather shallow and don't capture the intricacies of the image well Next is a deeper model with BatchNormalisation and Dropout. Using transferlearning, LeNet50 and ResNet50 (with imagenet weights) models are implemented taking on-the-fly augmented data, some training only selected layers at the end of the model. \n### Lastly, a deep CNN model taking in augmented data and working with decaying learning rate which gave the best result.\n### The epochs for the first 6 models are **very low as they were not the best performing models**. Hence their graphs do not really do justice to the actual performance due to less plotting points.\n\n## To skip to the bset performing model jump to **Cell 44**.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cv2\nfrom PIL import Image\nimport glob\nimport keras as k\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.optimizers import SGD\nfrom keras.layers import Flatten\nfrom keras.layers import Dense \nfrom keras.layers import Conv2D\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import AveragePooling2D\nfrom keras.optimizers import Adam\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Dropout \nfrom keras.layers import Activation\nimport keras\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.resnet50 import ResNet50\nfrom numpy import loadtxt\nfrom keras.models import Model\nfrom keras.callbacks import ReduceLROnPlateau\nimport tensorflow as tf","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain = train['label']\nxtrain = train.drop(labels = [\"label\"],axis = 1) \nxtrain.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain = xtrain / 255.0\ntest = test / 255.0\n\nxtrain.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain = xtrain.values.reshape(-1,28,28,1)                         #Reshaping into size(examples, height, width, channel)\nxtest = test.values.reshape(-1,28,28,1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(28*28)\nxtrain[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain = np.asarray(tf.one_hot(ytrain, 10, axis = -1)) \nytrain[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain[3]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xtrain.shape)\nprint(ytrain.shape)\nprint(xtest.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(xtrain, ytrain, test_size=0.1, random_state=42)\nprint(x_train.shape)\nprint(x_val.shape)\nprint(y_train.shape)\nprint(y_val.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We will use the below defined function after making prediction from the trained model, to create the final submission file and visualising the model performance.","metadata":{}},{"cell_type":"code","source":"def submission_and_visualization(final_predictions, model_number, model_history):\n    y = final_predictions.copy()\n    y= np.argmax(y, axis = 1)\n    y.reshape(28000,1);\n\n    col1 = np.arange(42000,70000,1)\n    col2 = y.copy()\n\n    final = np.stack((col1, col2), axis = 1)\n    finaldf = pd.DataFrame(data=final)\n    finaldf.rename(columns = {0:'filename', 1:'label'} , inplace = True)\n\n    name = []\n    for i in finaldf['filename']:\n        i = str(i)+\".png\"\n        name.append(i)\n\n    finaldf['filenames'] = name\n    finaldf['filename']=finaldf['filenames']\n    del finaldf['filenames']\n    \n    submission_file = \"Solution\"+str(model_number)+\".csv\"\n    finaldf.to_csv(submission_file, index=False)\n\n    plt.plot(model_history.history['loss'], label='Train loss')\n    plt.plot(model_history.history['val_loss'], label='Validation loss')\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Loss\")\n    plt.title(\"Loss Graph\")\n    plt.legend()\n    plt.show()\n    plt.plot(model_history.history['accuracy'], label = 'Train accuracy')\n    plt.plot(model_history.history['val_accuracy'], label = 'Validation accuracy')\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Accuracy\")\n    plt.title(\"Accuracy Graph\")\n    plt.legend()\n    plt.show()\n    \n    return ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN Models :","metadata":{}},{"cell_type":"markdown","source":"## Model 1 \n### Basic model without data augmentation, dropout, batch normalisation etc. ","metadata":{}},{"cell_type":"code","source":"model1 = Sequential()\nmodel1.add(Conv2D(16, (3,3), activation = 'relu', input_shape = (28,28,1)))\nmodel1.add(MaxPooling2D((2,2)))\nmodel1.add(Conv2D(32, (3,3), activation = 'relu'))\nmodel1.add(MaxPooling2D((2,2)))\nmodel1.add(Flatten())\nmodel1.add(Dense(100, activation = 'relu'))\nmodel1.add(Dense(64, activation = 'relu'))\nmodel1.add(Dense(10, activation = 'softmax'))\nmodel1.compile(optimizer = Adam(), loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model1.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history1 = []                  #Using callback to store history of accuracies and losses through and with which model proggresses. \nhistory1 = model1.fit(x_train, y_train, batch_size=32, epochs = 10, validation_data=(x_val, y_val),  callbacks=[history1]) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nif not os.path.isdir('models'):\n    os.mkdir('models')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import model_from_json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# serialize model to JSON\nmodel_json = model1.to_json()\nwith open(\"models/model.json\", \"w\") as json_file:\n    json_file.write(model_json)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# serialize weights to HDF5\nmodel1.save_weights(\"models/model.h5\")\nprint(\"Saved model to disk\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.remove('models/history1.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load json and create model\njson_file = open(\"models/model.json\", 'r')\nloaded_model_json = json_file.read()\njson_file.close()\nloaded_model = model_from_json(loaded_model_json)\n# load weights into new model\nloaded_model.load_weights(\"models/model.h5\")\nprint(\"Loaded model from disk\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest1 = loaded_model.predict(xtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtest.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtest[0:1].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest2 = loaded_model.predict(xtest[0:1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytest2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.argmax(ytest2, axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}