{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis notebook walks you through the implementation of Convolutional Neural Networks (CNN) to identify digits from the MNIST dataset - greyscale handwritten images. \\\nLet's start with some imports.","metadata":{}},{"cell_type":"code","source":"# imports\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.layers import Input, Dense, Activation, Flatten, Conv2D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import EarlyStopping","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-18T08:02:15.249199Z","iopub.execute_input":"2024-09-18T08:02:15.249588Z","iopub.status.idle":"2024-09-18T08:02:15.263628Z","shell.execute_reply.started":"2024-09-18T08:02:15.249548Z","shell.execute_reply":"2024-09-18T08:02:15.262831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"# converts label vector Y to one hot matrices\ndef convert_to_one_hot(Y, C):\n    # same as tf.keras.utils.to_categorical(Y_train_original , num_classes=10)\n    # Convert Pandas Series to a NumPy array\n    Y = np.array(Y)\n    Y = np.eye(C)[Y.reshape(-1)].T\n    return Y\n\n# plots a random sample of images from the training dataset\ndef plot_sample_images():\n    # Randomly select 9 sample indices\n    sample_indices = np.random.choice(len(X_train), 9, replace=False)\n    fig, axes = plt.subplots(3, 3, figsize=(10, 10))\n    \n    # Loop through each subplot\n    for i, ax in enumerate(axes.flat):\n        # Get the sample image and label\n        sample_image = X_train[sample_indices[i], :, :, 0]  # Grayscale images, so only 1 channel\n        sample_label = Y_train_visualization[sample_indices[i]]\n        \n        # Display the image\n        ax.imshow(sample_image, cmap='gray')\n        # Set the title with the label\n        ax.set_title(f'Label: {sample_label}')\n        # Hide the axes\n        ax.axis('off')\n\n    # Adjust layout\n    plt.tight_layout()\n    plt.show()\n    \n# plots the loss and accuracy of a trained model\ndef plot_loss_and_accuracy(history):\n    history_dict = history.history\n    # Extract the loss and accuracy from the history\n    loss = history_dict['loss']\n    accuracy = history_dict['accuracy']\n\n    # Extract the number of epochs\n    epochs = range(1, len(loss) + 1)\n\n    # Plot the training loss\n    plt.figure(figsize=(12, 4))\n\n    # Loss plot\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, loss, 'bo-', label='Training Loss')  # 'bo-' = blue dots with lines\n    plt.title('Training Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    # Accuracy plot\n    plt.subplot(1, 2, 2)\n    plt.plot(epochs, accuracy, 'go-', label='Training Accuracy')  # 'go-' = green dots with lines\n    plt.title('Training Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n\n    # Show the plots\n    plt.tight_layout()\n    plt.show()\n    \ndef plot_predictions(X, predicted_classes):\n    fig, axes = plt.subplots(3, 3, figsize=(10, 10))\n    \n    # Randomly select 9 sample indices\n    sample_indices = np.random.choice(len(X_test), 9, replace=False)\n    \n    # Loop through each subplot\n    for i, ax in enumerate(axes.flat):\n        # Get the sample image and predicted label\n        sample_image = X[sample_indices[i], :, :, 0]  # Greyscale images (1 channel)\n        predicted_label = predicted_classes[sample_indices[i]]\n        \n        # Display the image\n        ax.imshow(sample_image, cmap='gray')\n        # Set the title with the predicted label\n        ax.set_title(f'Predicted: {predicted_label}')\n        # Hide the axes\n        ax.axis('off')\n\n    # Adjust layout\n    plt.tight_layout()\n    plt.show()\n    \ndef plot_pixel_values(X_flatten):\n    # Plot the distribution of pixel values\n    plt.figure(figsize=(10, 6))\n    plt.hist(X_flatten, bins=100, range=(0, 1), color='red', alpha=0.7)\n    plt.title(\"Distribution of Pixel Intensities Across All Images\")\n    plt.xlabel(\"Pixel Intensity (0-1)\")\n    plt.ylabel(\"Frequency\")\n    plt.show()\n    \ndef top_pixel_intensities(X_flatten):\n    # Compute the histogram using numpy with a higher number of bins\n    hist_values, bin_edges = np.histogram(X_flatten, bins=10000, range=(0, 1))\n\n    # Find the indices of the top 3 values\n    top_5_indices = np.argsort(hist_values)[-5:][::-1]  # Sort and get the top 3 indices in descending order\n\n    # Extract the top 3 pixel intensities and their corresponding frequencies\n    top_5_values = []\n    for idx in top_5_indices:\n        intensity_value = (bin_edges[idx] + bin_edges[idx + 1]) / 2  # Find the middle of the bin\n        frequency = hist_values[idx]\n        top_5_values.append((intensity_value, frequency))\n\n    # Convert the result to a pandas DataFrame for a table display\n    top_5_df = pd.DataFrame(top_5_values, columns=['Pixel Intensity', 'Frequency'])\n\n    # Display the table\n    print(top_5_df)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:15.269752Z","iopub.execute_input":"2024-09-18T08:02:15.270099Z","iopub.status.idle":"2024-09-18T08:02:15.290931Z","shell.execute_reply.started":"2024-09-18T08:02:15.270065Z","shell.execute_reply":"2024-09-18T08:02:15.289845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data\n\nAn image is no more than a matrix of pixels. In this competition, the images provided are black and white and so each pixel is a greyscale value between 0 (total absence, black) and 255 (total presence, white), resulting in 256 possible values for each individual pixel of the image. \n\nEach image provided in the datasets is 28 pixels in height and 28 pixels in width, meaning that an image is comprised of 784 pixels in total. \n\nThe training dataset can be dissected into Y | X\nY is the vector of labels or digits, with a shape of (m,1). Y has values between 0 and 9, corresponding to the digital representation of the handwritten number, in this case an image.\nX is the features matrix, whereby each of the 784 columns corresponds to a pixel of the image. Therefore, X is of shape (m,784), where m is the number of sample images in the dataset.\n\n## Load datasets\n\nThe first main step is to load the datasets. Let's start by creating a function called load_data() to load and breakdown the training dataset into its respective labels (Y_train) and features (X_train). X_test will be used for making predictions later on. I'm also adding to this section some useful functions to be used later.","metadata":{}},{"cell_type":"code","source":"def load_data():\n    # set the input directory\n    input_dir = '/kaggle/input/digit-recognizer/'\n    \n    # load the dataset for training\n    train_dataset = pd.read_csv(input_dir+'train.csv')\n    # break down the training dataset into features and labels, ie X and Y\n    Y_train = train_dataset.iloc[:, 0]\n    X_train = train_dataset.iloc[:, 1:]\n    \n    # load the dataset for testing\n    X_test = pd.read_csv(input_dir+'test.csv')\n    \n    classes = 10 # 0 to 9\n    \n    return X_train, Y_train, X_test, classes\n\n# load the data\nX_train_original, Y_train_original, X_test_original, classes = load_data()","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:15.294958Z","iopub.execute_input":"2024-09-18T08:02:15.295241Z","iopub.status.idle":"2024-09-18T08:02:20.879258Z","shell.execute_reply.started":"2024-09-18T08:02:15.295209Z","shell.execute_reply":"2024-09-18T08:02:20.878210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transform datasets\n\nAfter loading the datasets, we want to standardize or normalize the data to have feature values between 0 and 1, to make it easier for the learning algorithm.\n\nWe're also changing the shape of X_test, Y_train, and X_test. This is done because the CNNs used below expect a 4D input shape with (batch_size, height, width, channels), in our case, (None, 28, 28, 1) for greyscale images.\n\nThe labels vector Y also needs to be turned into one hot matrices. For instance, a label/digit of '1',corresponds to a one hot matrix [0 1 0 0 0 0 0 0 0 0], and '9' = [0 0 0 0 0 0 0 0 1 0]. Notice that there are 10 positions, which matches the number of classes in this exercise ( digits from 0 to 9 = 10 digits/classes).\n\nFinally, we output some shape information about the datasets and plot some sample images from the training dataset.","metadata":{}},{"cell_type":"code","source":"X_train_original = X_train_original / 255. # as per competition details, max grayscale value is 255\nX_test_original = X_test_original / 255.\n#print(X_train.iloc[2000,400])\n\n# Training dataset\nX_train = X_train_original.to_numpy()\nX_train_visualization = X_train.copy()\nX_train = X_train.reshape(-1, 28, 28, 1)\nY_train_visualization = Y_train_original.copy()\n# Convert training labels to one hot matrices\nY_train = convert_to_one_hot(Y_train_original, classes).T\n\n# Test dataset\nX_test = X_test_original.to_numpy()\nX_test = X_test.reshape(-1, 28, 28, 1)\n\nm = X_train.shape[0]\nprint(f'Number of training images, m: {m}')\nprint(f'Number of test images: {X_test.shape[0]}')\nprint(f'X_train shape: {X_train.shape}')\nprint(f'X_train_vis shape: {X_train_visualization.shape}')\nprint(f'Y_train shape: {Y_train.shape}')\nprint(f'X_test shape: {X_test.shape}')","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:20.881103Z","iopub.execute_input":"2024-09-18T08:02:20.881429Z","iopub.status.idle":"2024-09-18T08:02:21.133935Z","shell.execute_reply.started":"2024-09-18T08:02:20.881396Z","shell.execute_reply":"2024-09-18T08:02:21.132944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"## Visualize samples","metadata":{}},{"cell_type":"code","source":"# plot random sample images with their labels as per training set\nplot_sample_images()","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:21.135078Z","iopub.execute_input":"2024-09-18T08:02:21.135397Z","iopub.status.idle":"2024-09-18T08:02:21.956960Z","shell.execute_reply.started":"2024-09-18T08:02:21.135362Z","shell.execute_reply":"2024-09-18T08:02:21.955971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next step is to count the number of times each class appears in the training set. This is called Frequency Distribution analysis, and it helps us telling how balanced the labels are in terms of samples to train the algorithm on.","metadata":{}},{"cell_type":"code","source":"labels_count = Y_train_visualization.value_counts()\nlabels_count_dict = {} # useful for the next step\nfor i in range(classes): # label values are [0,9]\n    count = labels_count.get(i, 0)  # returns 0 if the number doesn't exist\n    print(f\"{i} occurs {count} times in training set.\")\n    labels_count_dict[i] = count","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:21.959135Z","iopub.execute_input":"2024-09-18T08:02:21.959475Z","iopub.status.idle":"2024-09-18T08:02:21.977244Z","shell.execute_reply.started":"2024-09-18T08:02:21.959432Z","shell.execute_reply":"2024-09-18T08:02:21.976169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_min_value, labels_max_value = np.min(labels_count), np.max(labels_count)\nkey = next((k for k, v in labels_count_dict.items() if v == labels_min_value), None)\nprint(f'Label {key} has the least samples in training dataset.')\nkey = next((k for k, v in labels_count_dict.items() if v == labels_max_value), None)\nprint(f'Label {key} has the most samples in training dataset.')\nlabels_mean, labels_std_dev = np.mean(labels_count), np.std(labels_count)\nprint(f'Labels count has a mean={labels_mean} and std deviation={labels_std_dev}')","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:21.978298Z","iopub.execute_input":"2024-09-18T08:02:21.978581Z","iopub.status.idle":"2024-09-18T08:02:21.986012Z","shell.execute_reply.started":"2024-09-18T08:02:21.978549Z","shell.execute_reply":"2024-09-18T08:02:21.985007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Another common practice in Exploratory Data Analysis is to check for missing values in the datasets. Let's do that.","metadata":{}},{"cell_type":"code","source":"total = X_train_original.isnull().sum().sort_values(ascending=False)\npercent = (X_train_original.isnull().sum()/X_train_original.isnull().count()).sort_values(ascending=False)\nmissing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:02:21.987239Z","iopub.execute_input":"2024-09-18T08:02:21.987666Z","iopub.status.idle":"2024-09-18T08:02:22.165072Z","shell.execute_reply.started":"2024-09-18T08:02:21.987618Z","shell.execute_reply":"2024-09-18T08:02:22.164045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The training dataset does not contain any missing values. Good. ","metadata":{}},{"cell_type":"markdown","source":"# Models & Outputs\n## Basic Sequential CNN\nNow let's build a basic CNN without performing any data augmentation techniques. There's no need to re-invent the wheel so I'm going to use the Tensorflow framework and its Sequential API to build the CNN model.\n\nThe CNN model will follow this architecture: CONV1->POOL1->CONV2->POOL2->CONV3->FLAT->FC->FC->Softmax.\n\nIn terms of tensor shapes in the CNN, we input a (28,28,1) greyscale image in CONV1 and then we perform a convolution with a (3,3,1) filter (or kernel). Since we use 32 filters, the resulting shape is (28,28,32). \nAfter POOL1, the new spatial dimensions reduce to half hence the output is of shape (14,14,32), with the number of channels remain ingthe same (32). \nThe next CONV2 block does a convolution of the previous volume with 64x (3,3,32) filters. The output of CONV2 has shape (14,14,64). POOL2 further halves the spatial dimension and therefore the new shape is (7,7,64). \nFinally, in CONV3, 128 filters of dimension (3,3,64) are applied and the output of this layer will then have a shape of (7,7,128). This is then flattened and passed through two Fully Connected Layers, culminating in a softmax with 10 classes. ","metadata":{}},{"cell_type":"code","source":"# Create the Sequential model.\nmodel_cnn1 = models.Sequential()\n\n# Add a Conv2D layer for grayscale images with input shape (28, 28, 1). This is the input layer.\nmodel_cnn1.add(layers.Conv2D(filters=32, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu', input_shape=(28, 28, 1)))\n\n# Add hidden layers to the CNN.\nmodel_cnn1.add(layers.MaxPooling2D((2, 2)))\nmodel_cnn1.add(layers.Conv2D(64, (3, 3), padding='same', activation='relu'))\nmodel_cnn1.add(layers.MaxPooling2D((2, 2)))\nmodel_cnn1.add(layers.Conv2D(128, (3, 3), padding='same', activation='relu'))\n\n# Flatten and Dense layers. Last layer uses a softmax activation to perform the classification.\nmodel_cnn1.add(layers.Flatten())\nmodel_cnn1.add(layers.Dense(128, activation='relu'))\nmodel_cnn1.add(layers.Dense(classes, activation='softmax'))\n\n# compile the model\n#opt = tf.keras.optimizers.Adam(learning_rate=0.00015)\nmodel_cnn1.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# summary of the model\nmodel_cnn1.summary()\n\n# early stopping settings. to be applied to all fit methods\nearly_stopping = EarlyStopping(monitor='val_loss', patience=12, restore_best_weights=True)\n\n# let's train the model now using our training dataset\n#history_cnn1 = model_cnn1.fit(X_train, Y_train, epochs = 25, batch_size = 128, validation_split=0.2, callbacks=[early_stopping])\nhistory_cnn1 = model_cnn1.fit(X_train, Y_train, epochs = 30, batch_size = 128)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:34:35.441175Z","iopub.execute_input":"2024-09-18T08:34:35.441554Z","iopub.status.idle":"2024-09-18T08:35:14.434524Z","shell.execute_reply.started":"2024-09-18T08:34:35.441515Z","shell.execute_reply":"2024-09-18T08:35:14.433523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# And let's now check the accuracy of the trained model on the training set. \nplot_loss_and_accuracy(history_cnn1)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:22:48.674032Z","iopub.execute_input":"2024-09-18T08:22:48.674850Z","iopub.status.idle":"2024-09-18T08:22:49.302314Z","shell.execute_reply.started":"2024-09-18T08:22:48.674806Z","shell.execute_reply":"2024-09-18T08:22:49.301265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After ~6 epochs, this basic CNN achieves over 99% accuracy on the training set. How does it perform on the Test dataset? Let's find out.","metadata":{}},{"cell_type":"code","source":"# Make predictions using the trained CNN above.\npredictions_cnn1 = model_cnn1.predict(X_test, batch_size=128, verbose=2)\n\n# predictions_seq contains the predicted probabilities for each class\n# argmax returns the class with the highest predicted probability\npredicted_classes_cnn1 = np.argmax(predictions_cnn1, axis=1)\n\n# Call the function to visualize predictions\nplot_predictions(X_test, predicted_classes_cnn1)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:22:56.040835Z","iopub.execute_input":"2024-09-18T08:22:56.041221Z","iopub.status.idle":"2024-09-18T08:22:58.599113Z","shell.execute_reply.started":"2024-09-18T08:22:56.041180Z","shell.execute_reply":"2024-09-18T08:22:58.598155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# write the submission file with the predicted values and then submit to the competition.\nsubmission_df = pd.DataFrame({\n    'ImageId': range(1, len(predicted_classes_cnn1) + 1),\n    'Label': predicted_classes_cnn1\n})\nsubmission_df.to_csv('submission1.csv', index=False)\n\n# OPTIONAL\nmodel_cnn1.save('/kaggle/working/mnist-cnn1.h5')","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:23:03.246949Z","iopub.execute_input":"2024-09-18T08:23:03.247370Z","iopub.status.idle":"2024-09-18T08:23:03.338526Z","shell.execute_reply.started":"2024-09-18T08:23:03.247331Z","shell.execute_reply":"2024-09-18T08:23:03.337505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The sequential CNN model above performs with **99.196% accuracy** on the **Test dataset**, when epochs=25. You can improve a few cents if you increase the number of epochs. However, after ~45 epochs the model stops learning. \n\nFor reference later, this model has 896,906 parameters in total and a size of 3.42 MB. Not too bad. ","metadata":{}},{"cell_type":"markdown","source":"## Basic Sequential CNN with 3 channels input\nI'm pretty happy with the accuracy on the Test set of the first CNN model, but let's keep going and build a similar model but now using the Tensorflow Sequential API. Also let's simulate RGB channels in the input layer.","metadata":{}},{"cell_type":"code","source":"# Modify the input layer to accept greyscale images of size 28x28 (1 channel)\ninput_layer = layers.Input(shape=(28, 28, 1))  # Grayscale images\n# Create 3 replicas of the greyscale channel to simulate RGB input (3 channels)\nX = layers.Lambda(lambda x: tf.tile(x, [1, 1, 1, 3]))(input_layer)\n\nX = layers.Conv2D(filters=32, kernel_size=(3, 3), strides=(1, 1), padding='same', activation='relu')(X)\nX = layers.MaxPooling2D((2, 2))(X)\nX = layers.Conv2D(64, (3, 3), padding='same', activation='relu')(X)\nX = layers.MaxPooling2D((2, 2))(X)\nX = layers.Conv2D(128, (3, 3), padding='same', activation='relu')(X)\n\nX = layers.Flatten()(X)\nX = layers.Dense(128, activation='relu')(X)\noutput_layer = layers.Dense(10, activation='softmax')(X)\n\n# Create the final model\nmodel_cnn2 = models.Model(inputs=input_layer, outputs=output_layer)\n\n# Compile the model\nmodel_cnn2.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Summary of the model\nmodel_cnn2.summary()","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:35:31.641026Z","iopub.execute_input":"2024-09-18T08:35:31.641948Z","iopub.status.idle":"2024-09-18T08:35:31.725740Z","shell.execute_reply.started":"2024-09-18T08:35:31.641904Z","shell.execute_reply":"2024-09-18T08:35:31.724885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'm going to implement Early Stopping to avoid manually choosing the number of epochs. What this feature does is it monitors the validation loss or accuracy and stopps the training once the model stops improving. This is an efficient way to find the optimal number of epochs without overtraining.\n\nWhy is overtraining an issue? Well, training for too many epochs can lead to *overfitting*, where the model learns to perform well on the training data but generalizes poorly to new/test data.","metadata":{}},{"cell_type":"code","source":"# let's train the model now using our training dataset\nhistory_cnn2 = model_cnn2.fit(X_train, Y_train, epochs = 50, batch_size = 128, validation_split=0.2, callbacks=[early_stopping])","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:35:35.757098Z","iopub.execute_input":"2024-09-18T08:35:35.757471Z","iopub.status.idle":"2024-09-18T08:36:00.022270Z","shell.execute_reply.started":"2024-09-18T08:35:35.757427Z","shell.execute_reply":"2024-09-18T08:36:00.021471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model stops learning around epoch 19/50.  ","metadata":{}},{"cell_type":"code","source":"plot_loss_and_accuracy(history_cnn2)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:03:00.249744Z","iopub.execute_input":"2024-09-18T08:03:00.250120Z","iopub.status.idle":"2024-09-18T08:03:00.898960Z","shell.execute_reply.started":"2024-09-18T08:03:00.250085Z","shell.execute_reply":"2024-09-18T08:03:00.898057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions using the trained model\npredictions_cnn2 = model_cnn2.predict(X_test, batch_size=128, verbose=2)\n# The output `predictions` will be a NumPy array containing the predicted probabilities for each class\n# take the argmax to get the class with the highest predicted probability\npredicted_classes_cnn2 = np.argmax(predictions_cnn2, axis=1)\n# Call the function to visualize predictions\nplot_predictions(X_test, predicted_classes_cnn2)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:03:00.900318Z","iopub.execute_input":"2024-09-18T08:03:00.900727Z","iopub.status.idle":"2024-09-18T08:03:02.886797Z","shell.execute_reply.started":"2024-09-18T08:03:00.900672Z","shell.execute_reply":"2024-09-18T08:03:02.885715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    'ImageId': range(1, len(predicted_classes_cnn2) + 1),\n    'Label': predicted_classes_cnn2\n})\nsubmission_df.to_csv('submission2.csv', index=False)\n\n# OPTIONAL\nmodel_cnn2.save('/kaggle/working/mnist-cnn2.h5')","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:03:02.888036Z","iopub.execute_input":"2024-09-18T08:03:02.888352Z","iopub.status.idle":"2024-09-18T08:03:02.981476Z","shell.execute_reply.started":"2024-09-18T08:03:02.888317Z","shell.execute_reply":"2024-09-18T08:03:02.980559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The functional CNN model with 3 channels input performs with similar accuracy compared to the first CNN on the Test dataset. This model has 896,906 parameters in total and a size of 3.42 MB. Same size as the previous model.","metadata":{}},{"cell_type":"markdown","source":"## Pre-trained ResNet50\n\nLastly, I'm going to train a model called ResNet50, which is a classic CNN model with 50 hidden layers, previously trained against the ImageNet dataset.\n\nThe ResNet50 model typically expects the images to have three channels (RGB). However, our MNIST datasets are in greyscale, hence we need to modify the first convolutional layer of ResNet50 to accept a single-channel input, like we did in the previous model. To make this happen, I'm going to use the Tensorflow Functional API.\n\nApplying a convolution between a 28x28x1 greyscale image and a (3,3) filter with a stride of 1 and padding \"same\" (to keep the integrity of the images), we get 28x28x3 volume. These 3 images are replicas of the first.","metadata":{}},{"cell_type":"code","source":"# Load the ResNet50 model with weights, but without the top classification layer because we want to use ours\nbase_model = ResNet50(weights='imagenet', include_top=False)\n# Modify the input layer to accept greyscale images of size 28x28 (1 channel)\ninput_layer = layers.Input(shape=(28, 28, 1))  # Grayscale images\n# Create 3 replicas of the greyscale channel to simulate RGB input (3 channels)\nX = layers.Conv2D(filters=3, kernel_size=(3,3), strides=(1,1), padding='same')(input_layer)\n# Pass this through the ResNet50 base model\nX = base_model(X)\n# Output layer\nX = Flatten()(X)\noutput_layer = Dense(classes, activation='softmax')(X)\n# Create the final ResNet model\nmodel_resnet = Model(inputs=input_layer, outputs=output_layer)\n# Summary of the model\nmodel_resnet.summary()\n\n# Compile the model\nopt = tf.keras.optimizers.Adam(learning_rate=0.00015)\nmodel_resnet.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:37:10.169729Z","iopub.execute_input":"2024-09-18T08:37:10.170651Z","iopub.status.idle":"2024-09-18T08:37:11.408318Z","shell.execute_reply.started":"2024-09-18T08:37:10.170612Z","shell.execute_reply":"2024-09-18T08:37:11.407582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's train the model using the training dataset. ","metadata":{}},{"cell_type":"code","source":"# let's train the model now using the training dataset\n#history_resnet = model_resnet.fit(X_train, Y_train, epochs = 50, batch_size = 128, validation_split=0.2, callbacks=[early_stopping])\nhistory_resnet = model_resnet.fit(X_train, Y_train, epochs = 25, batch_size = 128)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:37:15.871388Z","iopub.execute_input":"2024-09-18T08:37:15.872268Z","iopub.status.idle":"2024-09-18T08:43:23.737157Z","shell.execute_reply.started":"2024-09-18T08:37:15.872209Z","shell.execute_reply":"2024-09-18T08:43:23.736191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# And let's now check the accuracy of the trained model against the training set. \nplot_loss_and_accuracy(history_resnet)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:43:28.882488Z","iopub.execute_input":"2024-09-18T08:43:28.883300Z","iopub.status.idle":"2024-09-18T08:43:29.462717Z","shell.execute_reply.started":"2024-09-18T08:43:28.883257Z","shell.execute_reply":"2024-09-18T08:43:29.461782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"But is it better in the Test dataset? Now let's make some predictions.","metadata":{}},{"cell_type":"code","source":"##### Make predictions using the trained model\npredictions_resnet = model_resnet.predict(X_test, batch_size=128, verbose=2)\npredicted_classes_resnet = np.argmax(predictions_resnet, axis=1)\n# visualize predictions\nplot_predictions(X_test, predicted_classes_resnet)","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:43:34.810363Z","iopub.execute_input":"2024-09-18T08:43:34.811260Z","iopub.status.idle":"2024-09-18T08:43:43.707944Z","shell.execute_reply.started":"2024-09-18T08:43:34.811216Z","shell.execute_reply":"2024-09-18T08:43:43.707042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, we create the submission csv file and write our predictions to it.","metadata":{}},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    'ImageId': range(1, len(predicted_classes_resnet) + 1),\n    'Label': predicted_classes_resnet\n})\nsubmission_df.to_csv('submission3.csv', index=False)\n\n# OPTIONAL\nmodel_resnet.save('/kaggle/working/mnist-resnet50.h5')","metadata":{"execution":{"iopub.status.busy":"2024-09-18T08:43:53.141050Z","iopub.execute_input":"2024-09-18T08:43:53.142029Z","iopub.status.idle":"2024-09-18T08:43:54.377769Z","shell.execute_reply.started":"2024-09-18T08:43:53.141984Z","shell.execute_reply":"2024-09-18T08:43:54.376593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The accuracy of the ResNet model on the Test set was **99.221%**. Notice that the model size is much larger than the previous ones. The addition of the pre-trained ResNet model jumps the total parameters to 23,608,232 (90.06 MB), a 26x jump! ","metadata":{}},{"cell_type":"markdown","source":"# Final remarks\nThe 3 models have a very similar accuracy on the Test dataset without any data preprocessing techniques. \n\nI can conclude that for this particular MNIST dataset, ResNet50 is not a good *practical* option because simpler CNN models have the same level of performance and are more resources friendly.","metadata":{}}]}