{"cells":[{"metadata":{},"cell_type":"markdown","source":"## There are three models implemented in this notebook. \n- MLP model, simple CNN model and ResNet18 model."},{"metadata":{},"cell_type":"markdown","source":"## Import Section"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\n\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# Seeding for reproducibility\nseed = 10\nnp.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read and prepare datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(path, train = False):\n    \"\"\"\n    This function will read the data from the given path \n    into a pandas dataframe, convert the datatype into float \n    and separate the features and labels from the dataframe.\n    \n    Args:\n        path: path to the dataset (string)\n        train: flag to indicate training/test data (binary)\n    \n    Returns:\n        features and labels\n    \"\"\"\n    df = pd.read_csv(path)\n    \n    if train:\n        # Separating features and labels from the training data\n        features = df.iloc[:, 1:].values.astype('float32')\n        labels = df.iloc[:, 0].values.astype('int16')\n        print(f'Shape of training features: {features.shape}')\n        print(f'Shape of training labels: {labels.shape}')\n        return features, labels\n    \n    else:\n        # For test data, there are only features\n        features = df.values.astype('float32')\n        print(f'Shape of test features: {features.shape}')\n        return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define training and test data paths\ntrain_path = '/kaggle/input/digit-recognizer/train.csv'\ntest_path = '/kaggle/input/digit-recognizer/test.csv'\n\nX, y = load_data(train_path, train = True)\nX_test = load_data(test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define global variables\nSHAPE = (28, 28)\nBATCH_SIZE = 64\nEPOCHS = 20\nNUM_CLASSES = 10","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display few images from training dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create figure object\nfig = plt.figure(figsize = (10, 10))\nfor i in range(16):\n    ax = fig.add_subplot(4, 4, i + 1)\n    ax.imshow(X[i].reshape(SHAPE), cmap = 'gray')\n    plt.axis('off')\n    plt.title('label: ' + str(y[i]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_data(features, shape):\n    \"\"\"\n    This function will reshape the features into the given shape\n    \n    Args:\n        features: numpy array of shape (N, 784)\n        shape: (28, 28) tuple\n        \n    Returns:\n        reshaped features (N, 28, 28, 1)\n    \"\"\"\n    # Explicitly adding channel dimension to the image data \n    features = features.reshape(-1, shape[0], shape[1], 1)\n    print(f'Shape of features: {features.shape}')\n    \n    features = features / 255.0\n    return features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MLP model for classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_mlp_model(num_hidden_layers = 1, units = None, drop_out = False, rate = 0.3):\n    \"\"\"\n    This function will create MLP model with given number of hidden\n    layers and units in each of these layers\n    \n    Args:\n        num_hidden_layers: number of hidden layers in the model (default = 1)\n        units: number of units in each hidden layer (default will be 128)\n        drop_out: flag to indicate dropout layer (default False)\n        rate: drop out rate to be used in dropout layer (default = 0.3)\n    Returns:\n        functional model\n    \"\"\"\n    inputs = layers.Input(shape = (784, ), name = 'input_layer')\n    if num_hidden_layers == 1:\n        x = layers.Dense(128, activation = 'relu', name = 'hidden_1')(inputs)\n        \n    else:\n        x = layers.Dense(units[0], activation = 'relu', name = 'hidden_1')(inputs)\n        for i in range(1, num_hidden_layers):\n            if drop_out:\n                x = layers.Dropout(rate)(x)\n            x = layers.Dense(units[i], activation = 'relu', name = 'hidden_' + str(i + 1))(x)\n            \n    output = layers.Dense(NUM_CLASSES, activation = 'softmax', name = 'output')(x)\n    model = tf.keras.Model(inputs, output, name = 'MLP')\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create mlp model with one hidden layer\nmlp_model = create_mlp_model(num_hidden_layers = 2,\n                             units = [256, 128],\n                             drop_out = True,\n                             rate = 0.25)\n\n# print the summary of the model\nmlp_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train and validation split"},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_data(x, y, split = 0.15):\n    \"\"\"\n    This function will split the data into training and \n    validation data set with given split size\n    \n    Args:\n        x: input features\n        y: labels\n        split: split size for validation set (default = 0.15)\n        \n    Returns:\n        train_x, train_y, val_x, val_y\n    \"\"\"\n    \n    train_x, val_x, train_y, val_y = train_test_split(x, y, test_size = split, random_state = 42, stratify = y)\n    print(f'Shape of train_x: {train_x.shape}\\nShape of train_y: {train_y.shape}')\n    print(f'Shape of val_x: {val_x.shape}\\nShape of val_y: {val_y.shape}')\n    \n    return train_x, train_y, val_x, val_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check the shape of training and validation dataset\ntrain_x, train_y, val_x, val_y = split_data(X, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compile and train the MLP model"},{"metadata":{"trusted":true},"cell_type":"code","source":"mlp_model.compile(loss = 'sparse_categorical_crossentropy',\n                  optimizer = tf.keras.optimizers.Adam(),\n                  metrics = ['accuracy'])\n\nmlp_model.fit(train_x, train_y, epochs = EPOCHS,\n              batch_size = BATCH_SIZE, \n              validation_data = (val_x, val_y))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- **MLP model** achieves an accuracy of **96.57** on the validation set.\n- Let us build a CNN model to improve the accuracy further."},{"metadata":{},"cell_type":"markdown","source":"## CNN model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_cnn_model():\n    # input layer \n    inputs = layers.Input(shape = SHAPE + (1, ))\n    \n    # Block 1: Conv--> BN--> MaxPool--> Dropout\n    x = layers.Conv2D(64, (5, 5), activation = 'relu', \n                      padding = 'same', name = 'conv_1')(inputs)\n    x = layers.BatchNormalization(name = 'bnorm_1')(x)\n    x = layers.MaxPooling2D(pool_size = (2, 2), name = 'mpool_1')(x)\n    x = layers.Dropout(0.25, name = 'dropout_1')(x)\n    \n    # Block 2: Conv--> BN--> MaxPool--> Dropout\n    x = layers.Conv2D(128, (3, 3), activation = 'relu', \n                      padding = 'same', name = 'conv_2')(x)\n    x = layers.BatchNormalization(name = 'bnorm_2')(x)\n    x = layers.MaxPooling2D(pool_size = (2, 2), name = 'mpool_2')(x)\n    x = layers.Dropout(0.25, name = 'dropout_2')(x)\n    \n    # Block 3: Conv--> BN--> MaxPool--> Dropout\n    x = layers.Conv2D(256, (3, 3), activation = 'relu', \n                      padding = 'same', name = 'conv_3')(x)\n    x = layers.BatchNormalization(name = 'bnorm_3')(x)\n    x = layers.MaxPooling2D(pool_size = (2, 2), name = 'mpool_3')(x)\n    x = layers.Dropout(0.25, name = 'dropout_3')(x)\n\n    # flatten the outputs\n    x = layers.Flatten(name = 'flatten')(x)\n    \n    # Block 4: Dense--> BN--> Droupout\n    x = layers.Dense(128, activation = 'relu', name = 'dense_1')(x)\n    x = layers.BatchNormalization(name = 'bnorm_4')(x)\n    x = layers.Dropout(0.2, name = 'dropout_4')(x)\n\n    # Block 5: Dense--> BN--> Droupout\n    x = layers.Dense(64, activation = 'relu', name = 'dense_2')(x)\n    x = layers.BatchNormalization(name = 'bnorm_5')(x)\n    x = layers.Dropout(0.3, name = 'dropout_5')(x)\n    outputs = layers.Dense(NUM_CLASSES, activation = 'softmax', name = 'output')(x)\n\n    # create model \n    model = tf.keras.Model(inputs, outputs, name = 'cnn_model')\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print the summary of the model\ncnn_model = create_cnn_model()\ncnn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare data for CNN model\nX = preprocess_data(X, shape = SHAPE)\n\n# split data into train and validation sets\ntrain_x, train_y, val_x, val_y = split_data(X, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compile and fit the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_model.compile(loss = 'sparse_categorical_crossentropy',\n                  optimizer = tf.keras.optimizers.Adam(),\n                  metrics = ['accuracy'])\n\ncnn_model.fit(train_x, train_y, epochs = EPOCHS,\n              batch_size = BATCH_SIZE // 2,\n              validation_data = (val_x, val_y))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- CNN model achieved 99% plus accuracy."},{"metadata":{},"cell_type":"markdown","source":"## ResNet18"},{"metadata":{"trusted":true},"cell_type":"code","source":"class IdentityBlock(tf.keras.Model):\n    \"\"\"\n    This class will create an identity block for the resnet model\n    by subclassing Model class\n    \"\"\"\n    def __init__(self, filters, kernel_size):\n        super(IdentityBlock, self).__init__(name = \"\")\n\n        self.conv1 = layers.Conv2D(filters, kernel_size, \n                                   padding = 'same')\n        self.bn1 = layers.BatchNormalization()\n        \n        self.conv2 = layers.Conv2D(filters, kernel_size, \n                                   padding = 'same')\n        self.bn2 = layers.BatchNormalization()\n        \n        self.act = layers.Activation('relu')\n        self.add = layers.Add()\n        \n        \n    def call(self, input_tensor):\n        # Block 1: Conv--> BN--> ReLU\n        x = self.conv1(input_tensor)\n        x = self.bn1(x)\n        x = self.act(x)\n        \n        # Block 2: Conv--> BN--> ReLU\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.act(x)\n        \n        # skip connection\n        x = self.add([x, input_tensor])\n        x = self.act(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet18(tf.keras.Model):\n    def __init__(self, num_classes):\n        super(ResNet18, self).__init__()\n        self.conv = layers.Conv2D(64, 7, padding = 'same')\n        self.bn = layers.BatchNormalization()\n        self.act = layers.Activation('relu')\n        self.max_pool = layers.MaxPool2D((3, 3))\n        self.drop_out = layers.Dropout(0.3)\n        \n        # create Identity blocks\n        self.id1 = IdentityBlock(64, 3)\n        self.id2 = IdentityBlock(64, 3)\n        \n        self.global_pool = layers.GlobalAveragePooling2D()\n        self.classifier = layers.Dense(num_classes, activation = 'softmax')\n        \n    def call(self, inputs):\n        x = self.conv(inputs)\n        x = self.bn(x)\n        x = self.act(x)\n        x = self.max_pool(x)\n        x = self.drop_out(x)\n        \n        # call identity blocks\n        x = self.id1(x)\n        x = self.drop_out(x)\n        x = self.id2(x)\n        x = self.drop_out(x)\n        \n        x = self.global_pool(x)\n        return self.classifier(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compile and train ResNet18 model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create instance of resnet model\nresnet18 = ResNet18(NUM_CLASSES)\n\nresnet18.compile(loss = 'sparse_categorical_crossentropy',\n                  optimizer = tf.keras.optimizers.Adam(),\n                  metrics = ['accuracy'])\n\nresnet18.fit(train_x, train_y, epochs = EPOCHS,\n              batch_size = BATCH_SIZE,\n              validation_data = (val_x, val_y))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make predictions on test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare test data\nX_test = preprocess_data(X_test, shape = SHAPE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making predictions on test data\npredictions = cnn_model.predict(X_test, verbose = 0)\n\nsubmissions = pd.DataFrame({\"ImageId\": list(range(1, len(predictions) + 1)),\n                            \"Label\": np.argmax(predictions, axis = -1)})\nsubmissions.to_csv(\"sm.csv\", index = False, header = True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}