{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Thanks to -\n1. https://xu932.github.io/blog/kaggle/2020-03-29-digit-recognizer/","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# 1. Importing required modules","execution_count":null},{"metadata":{"_uuid":"65b31664-15a4-4676-b13c-02b99a48761e","_cell_guid":"87e65770-f644-45f4-8f5b-0f5b8f24a4e0","trusted":true,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf # tensorflow\nimport matplotlib.pyplot as plt # Matplotlib\nimport seaborn as sns # Seaborn\nfrom sklearn.preprocessing import StandardScaler # Feature scaling\nfrom sklearn.model_selection import learning_curve, RandomizedSearchCV, GridSearchCV # Searching best parameters\nfrom sklearn.model_selection import StratifiedKFold, cross_validate, train_test_split # Cross validation\nfrom datetime import datetime # Date functions\nfrom scipy.ndimage import shift, rotate # Image augmentation\nfrom tensorflow.keras import Input, Model\nimport keras\nfrom tensorflow.keras.layers import ReLU\nfrom keras.layers import Dropout, Flatten, Dense, LeakyReLU, Conv2D, MaxPool2D, BatchNormalization\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom keras.models import load_model\n\n\n\nSEED = 20 # Random state\n%matplotlib inline\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Importing data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing training and test data\n\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Shape of test and train set\n\nprint(f\"Training set = {train.shape}\")\nprint(f\"Testing set = {test.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training set view...............\\n\")\n\nprint(train.iloc[:,:8].head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Data pre-processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Standardising the dataset\n\nX = train.loc[:,train.columns != 'label'] / 255\nxTest = test / 255\ny = train.label\n\nxTrain, xVal, yTrain, yVal = train_test_split(X, y, random_state = SEED, stratify = y, test_size = 0.25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Class containing functions to augment images\n\nclass imageAugmentation():\n    \n    # Function to shift pixels of image in desired direction\n    '''\n    Input: Data, desired position \n    Ouput: Processed data as an array\n    '''\n    def shiftPixel(self, data, position = \"left\"):\n        \n        image = []\n        \n        if position == \"left\":\n            for index in range(data.shape[0]):\n                row = data.iloc[index].values.reshape(28,28)\n                image.append(shift(row, [0,-1]))\n            return np.array(image)\n        \n        elif position == \"right\":\n            for index in range(data.shape[0]):\n                row = data.iloc[index].values.reshape(28,28)\n                image.append(shift(row, [0,1]))\n            return np.array(image)\n        \n        elif position == \"up\":\n            for index in range(data.shape[0]):\n                row = data.iloc[index].values.reshape(28,28)\n                image.append(shift(row, [-1,0]))\n            return np.array(image)\n        \n        elif position == \"down\":\n            for index in range(data.shape[0]):\n                row = data.iloc[index].values.reshape(28,28)\n                image.append(shift(row, [1,0]))\n            return np.array(image)\n        \n        \n    # Function to rotate image\n    '''\n    Input: Data, direction in which image is to be rotated\n    Output: Processed data as array\n    '''\n    def rotateImage(self, data, direction = \"clockwise\"):\n        \n        image = []\n        \n        for index in range(data.shape[0]):\n            \n            row = data.iloc[index].values.reshape(28,28)\n            \n            if direction == \"clockwise\":\n                image.append(rotate(row, -10, reshape=False))\n        \n            elif direction == \"anti_clockwise\":\n                image.append(rotate(row, -10, reshape=False))\n                \n        return np.array(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentImage = imageAugmentation()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start = datetime.now()\naugmentedData = xTrain.values.reshape(-1,28,28)\naugmentedData = np.r_[augmentedData, augmentImage.shiftPixel(xTrain, position = \"left\")]\naugmentedData = np.r_[augmentedData, augmentImage.shiftPixel(xTrain, position = \"right\")]\naugmentedData = np.r_[augmentedData, augmentImage.shiftPixel(xTrain, position = \"up\")]\naugmentedData = np.r_[augmentedData, augmentImage.shiftPixel(xTrain, position = \"down\")]\naugmentedData = np.r_[augmentedData, augmentImage.rotateImage(xTrain)]\naugmentedData = np.r_[augmentedData, augmentImage.rotateImage(xTrain, direction = \"anti_clockwise\")]\nprint(f\"Total time taken to apply shift to images = {datetime.now() - start}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentedLabels = np.r_[yTrain,yTrain,yTrain,yTrain,yTrain,yTrain,yTrain]\nprint(augmentedData.shape)\nprint(augmentedLabels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualising image\n\nximagelist = augmentedData[150000].reshape(28,28)\nim = plt.imshow(ximagelist, cmap='Greys')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# 4. Modelling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 4.1. Simple CNN model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting label encoding for output labels\ntrainLabels = to_categorical(\n    augmentedLabels, num_classes=10, dtype='float32'\n)\nvalidationLabels = to_categorical(\n    yVal, num_classes=10, dtype='float32'\n)\nvalidationData = xVal.values.reshape(-1, 28, 28)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xTest = xTest.values.reshape(-1, 28, 28)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Simple CNN model\n\ninputs = Input(shape = (28,28))\nX = Flatten()(inputs)\nX = Dense(100, activation = LeakyReLU(alpha = 0.3))(X)\nX = Dropout(rate = 0.5)(X)\nX = Dense(50, activation = LeakyReLU(alpha = 0.3))(X)\nX = Dropout(rate = 0.3)(X)\noutputs = Dense(10, activation = \"softmax\")(X)\nmodel = Model(inputs, outputs, name = \"SimpleCNN\")\ncallback = EarlyStopping(monitor='loss', patience=3)\n\nmodel.compile(loss=\"categorical_crossentropy\",\n   optimizer=RMSprop(lr=1e-4),\n   metrics=[\"acc\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model summary\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Training the model\n\nstart = datetime.now()\n\nhistory = model.fit(\n    augmentedData, \n    trainLabels, \n    epochs=150,\n    batch_size=32,\n    callbacks=[callback],\n    validation_data = (validationData, validationLabels)\n)\n\nprint(f\"\\nTotal training time = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of optimum epochs\n\nprint(f\"Number of optimal epochs = {len(history.history['loss'])}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model evaluation the validation set\n\nstart = datetime.now()\n\n(evaluationLoss, evaluationAccuracy) = model.evaluate(\n    validationData, \n    validationLabels, \n    batch_size=32,\n    verbose=1\n)\n\nprint(f\"\\nTotal evaluation time = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 4.2. CNN model with convolution and pooling layer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Simple CNN model\n\ndef complexCNNModel():\n    inputs = Input(shape = (28,28,1))\n    X = Conv2D(filters=64, kernel_size=5)(inputs)\n    X = ReLU()(X)\n    X = Dropout(rate = 0.5)(X)\n    X = MaxPool2D(pool_size=(2, 2))(X)\n    X = Conv2D(filters=256, kernel_size=3)(X)\n    X = ReLU()(X)\n    X = Dropout(rate = 0.5)(X)\n    X = MaxPool2D(pool_size=(2, 2))(X)\n    X = Dense(units=512)(X)\n    X = ReLU()(X)\n    X = Dropout(rate = 0.5)(X)\n    X = Dense(units=256)(X)\n    X = ReLU()(X)\n    X = Dropout(rate = 0.5)(X)\n    X = Dense(units=128)(X)\n    X = Flatten()(X)\n    outputs = Dense(10, activation = \"softmax\")(X)\n    model = Model(inputs, outputs, name = \"Complex_CNN\")\n    callback = EarlyStopping(monitor='loss', patience=3)\n\n    model.compile(loss=\"categorical_crossentropy\",\n       optimizer=RMSprop(lr=1e-4),\n       metrics=[\"acc\"])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model summary\n\nmodel = complexCNNModel()\ncallback = EarlyStopping(monitor='loss', patience=3)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training the model\n\nstart = datetime.now()\n\nhistory = model.fit(\n    augmentedData, \n    trainLabels, \n    epochs=150,\n    batch_size=32,\n    callbacks=[callback],\n    validation_data = (validationData, validationLabels)\n)\n\nprint(f\"\\nTotal training time = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of optimum epochs\n\nprint(f\"Number of optimal epochs = {len(history.history['loss'])}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model evaluation the validation set\n\nstart = datetime.now()\n\n(evaluationLoss, evaluationAccuracy) = model.evaluate(\n    validationData, \n    validationLabels, \n    batch_size=32,\n    verbose=1\n)\n\nprint(f\"\\nTotal evaluation time = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making predictions\n\nxTest = xTest.values.reshape(-1, 28, 28)\n\n# predictions = model.predict(xTest)\n# labels = predictions.argmax(axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to write predictions into a csv file\n'''\nInput: Predictions, filename, ID\nOutput: Generates a csv file\n'''\ndef pred_to_csv(predictions, file_name):\n    \n    submissions = pd.DataFrame(columns = [\"ImageId\", \"Label\"])\n    submissions.ImageId = test.index + 1\n    submissions.Label = predictions\n    submissions.to_csv('{}.csv'.format(file_name), header=True, index=False)\n    \npred_to_csv(labels, \"CNN\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 4.3. Ensemble of 10 models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ensembles = []\n\nfor i in range(10):\n    model = complexCNNModel()\n    ensembles.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Training the model\n\nstart = datetime.now()\n\nhistory = []\ncallback = EarlyStopping(monitor='loss', patience=3)\n\nfor model in ensembles:\n\n    history.append(model.fit(\n        augmentedData, \n        trainLabels, \n        epochs=150,\n        batch_size=32,\n        callbacks=[callback],\n        validation_data = (validationData, validationLabels)\n    ))\n    \n#     break\n\nprint(f\"\\nTotal training time for 10 models = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saving models in h5 format\n\nfor index, model in enumerate(ensembles,1):\n    model.save(f\"CNNmodel{index}.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making predictions\n\nensembleResults = np.zeros((xTest.shape[0],10)) \nfor i in range(10):\n    ensembleResults = ensembleResults + ensembles[i].predict(xTest)\nensembleResults = np.argmax(ensembleResults,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating csv file and writing predictions\n\npred_to_csv(ensembleResults, \"ensembleCNN\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 4.4. LeNet model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to create a LeNet CNN model\n\ndef leNet():\n    inputs = Input(shape = (28,28,1))\n    \n    X = Conv2D(filters=32, kernel_size=3, activation = \"relu\")(inputs)\n    X = BatchNormalization()(X)\n    X = Conv2D(filters=32, kernel_size=3, activation = \"relu\")(X)\n    X = Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu')(X)\n    X = BatchNormalization()(X)\n    X = Dropout(0.4)(X)\n    \n    X = Conv2D(filters=64, kernel_size=3, activation = \"relu\")(X)\n    X = BatchNormalization()(X)\n    X = Conv2D(filters=64, kernel_size=3, activation = \"relu\")(X)\n    X = Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu')(X)\n    X = BatchNormalization()(X)\n    X = Dropout(0.4)(X)\n    \n    X = Conv2D(128, kernel_size = 4, activation='relu')(X)\n    X = BatchNormalization()(X)\n    X = Flatten()(X)\n    X = Dropout(0.4)(X)\n    outputs = Dense(10, activation = \"softmax\")(X)\n    model = Model(inputs, outputs, name = \"leNet\")\n    \n    model.compile(loss=\"categorical_crossentropy\",\n       optimizer= Adam(),\n       metrics=[\"acc\"])\n    \n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"leNet = leNet()\nleNet.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Training leNet model\n\nstart = datetime.now()\n\ncallback = EarlyStopping(monitor='loss', patience=3)\nhistory = leNet.fit(\n        augmentedData, \n        trainLabels, \n        epochs=150,\n        batch_size=32,\n        callbacks=[callback],\n        validation_data = (validationData, validationLabels)\n    )\n    \nprint(f\"\\nTotal training time for LeNet model = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = leNet.predict(xTest)\nlabels = predictions.argmax(axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating csv file and writing predictions\n\npred_to_csv(labels, \"lenet\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.5. Ensemble leNet models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ensembles = []\n\nfor i in range(5):\n    model = leNet()\n    ensembles.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training the model\n\nstart = datetime.now()\n\nhistory = []\ncallback = EarlyStopping(monitor='loss', patience=3)\n\nfor model in ensembles:\n\n    history.append(model.fit(\n        augmentedData, \n        trainLabels, \n        epochs=150,\n        batch_size=32,\n        callbacks=[callback],\n        validation_data = (validationData, validationLabels)\n    ))\n    \n#     break\n\nprint(f\"\\nTotal training time for 5 models = {datetime.now()-start}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saving models\n\nfor index, model in enumerate(ensembles,6):\n    model.save(f\"leNet{index}.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing leNet model\nleNetModels = []\nfor i in range(10):\n    leNetModels.append(load_model(f\"../input/lenetmodels/leNet{i+1}.h5\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making predictions\nlenetEnsembleResults = np.zeros((xTest.shape[0],10)) \nfor i in range(10):\n    lenetEnsembleResults = lenetEnsembleResults + leNetModels[i].predict(xTest)\nlenetEnsembleResults = np.argmax(lenetEnsembleResults,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_to_csv(lenetEnsembleResults, \"ensembleLeNet\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}