{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"},{"sourceId":15444,"sourceType":"datasetVersion","datasetId":11102},{"sourceId":1782442,"sourceType":"datasetVersion","datasetId":1059701}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <div style=\"padding: 30px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#FFCE30\"><b><span style='color:#FFFFFF'>1 |</span></b> <b>INTRODUCTION</b></div>\n\n### <b><span style='color:#FFCE30'> 1.1 |</span> Description</b>\nThe CIFAR-10 dataset consists of 60,000 32x32 color images in 10 different classes, with 6,000 images per class. The dataset is divided into 50,000 training images and 10,000 testing images. The classes are mutually exclusive and there is no overlap between them.\nIn this notebook, we attempt to use Multi-Layer Perceptron (MLP), Convolutional Neural Network (CNN) and Transfer Learning (ResNet50 - TensorFlow) to build a model than can recognise the image of the test set after being trained on training set from CIFAR-10.\n\n### <b><span style='color:#FFCE30'> 1.2 |</span> Features</b>\n- Number of Instances: 60,000 (50,000 for training, 10,000 for testing)\n- Image Size: 32x32 pixels, color images (3 color channels)\n- Number of Classes: 10\n\n### <b><span style='color:#FFCE30'> 1.3 |</span> Accessibility</b>\nThe CIFAR-10 dataset is widely used for machine learning and computer vision research and can be easily accessed through libraries like TensorFlow and PyTorch.\n","metadata":{"jp-MarkdownHeadingCollapsed":true}},{"cell_type":"markdown","source":"# <div style=\"padding: 30px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#FFCE30\"><b><span style='color:#FFFFFF'>2 |</span></b> <b>LOAD AND EXPLORATION</b></div>","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 2.1 |</span> Load</b>","metadata":{}},{"cell_type":"code","source":"# pip install tensorflow","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:28:53.322713Z","iopub.execute_input":"2024-03-31T14:28:53.323122Z","iopub.status.idle":"2024-03-31T14:28:53.328548Z","shell.execute_reply.started":"2024-03-31T14:28:53.323088Z","shell.execute_reply":"2024-03-31T14:28:53.327426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:28:53.330717Z","iopub.execute_input":"2024-03-31T14:28:53.331366Z","iopub.status.idle":"2024-03-31T14:29:07.706592Z","shell.execute_reply.started":"2024-03-31T14:28:53.331334Z","shell.execute_reply":"2024-03-31T14:29:07.705664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:07.707846Z","iopub.execute_input":"2024-03-31T14:29:07.708493Z","iopub.status.idle":"2024-03-31T14:29:07.970279Z","shell.execute_reply.started":"2024-03-31T14:29:07.708457Z","shell.execute_reply":"2024-03-31T14:29:07.969051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the CIFAR-10 dataset\ncifar10 = tf.keras.datasets.cifar10\n(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n\n# Exploring the dataset\nprint(\"Training set shape:\", x_train.shape)\nprint(\"Test set shape:\", x_test.shape)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:07.971884Z","iopub.execute_input":"2024-03-31T14:29:07.972284Z","iopub.status.idle":"2024-03-31T14:29:13.427603Z","shell.execute_reply.started":"2024-03-31T14:29:07.972249Z","shell.execute_reply":"2024-03-31T14:29:13.426623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train Images Shape:      ', x_train.shape)\nprint('Train Labels Shape:      ', y_train.shape)\n\nprint('\\nTest Images Shape:       ', x_test.shape)\nprint('Test Labels Shape:       ', y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:13.431074Z","iopub.execute_input":"2024-03-31T14:29:13.431480Z","iopub.status.idle":"2024-03-31T14:29:13.437990Z","shell.execute_reply.started":"2024-03-31T14:29:13.431453Z","shell.execute_reply":"2024-03-31T14:29:13.436861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 2.2 |</span> Explore</b>","metadata":{}},{"cell_type":"code","source":"# Exploring the CIFAR-10 dataset with and overview\n\n# CIFAR-10 classes\nclass_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n               \n# Create a new figure\nplt.figure(figsize=(15,15))\n\n# Loop over the first 25 images\nfor i in range(64):\n    # Create a subplot for each image\n    plt.subplot(8, 8, i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n\n    # Display the image\n    plt.imshow(x_train[i])\n\n    # Set the label as the title\n    plt.title(class_names[y_train[i][0]], fontsize=12)\n\n# Display the figure\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:13.439212Z","iopub.execute_input":"2024-03-31T14:29:13.439486Z","iopub.status.idle":"2024-03-31T14:29:16.703691Z","shell.execute_reply.started":"2024-03-31T14:29:13.439463Z","shell.execute_reply":"2024-03-31T14:29:16.702698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define class names\nclasses_name = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n\n# Get class distribution for training and testing sets\ntrain_classes, train_counts = np.unique(y_train, return_counts=True)\ntest_classes, test_counts = np.unique(y_test, return_counts=True)\n\n# Set figure size and create subplots\nplt.figure(figsize=(15, 5))\n\n# Plot class distribution for training set\nplt.subplot(1, 2, 1)\nplt.barh(classes_name, train_counts, color='blue')\nplt.xlabel('Number of Samples')\nplt.title('Class Distribution in Training Set')\n\n# Plot class distribution for testing set\nplt.subplot(1, 2, 2)\nplt.barh(classes_name, test_counts, color='orange')\nplt.xlabel('Number of Samples')\nplt.title('Class Distribution in Testing Set')\n\n# Adjust layout and display the plot\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:16.704874Z","iopub.execute_input":"2024-03-31T14:29:16.705130Z","iopub.status.idle":"2024-03-31T14:29:17.319734Z","shell.execute_reply.started":"2024-03-31T14:29:16.705107Z","shell.execute_reply":"2024-03-31T14:29:17.318689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"padding: 30px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#FFCE30\"><b><span style='color:#FFFFFF'>3 |</span></b> <b>DATA NORMALIZATION</b></div>\n\n","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 3.1 |</span> Normalization to [0,1]</b>\n- Compute the minimum and maximum pixel values in the training dataset.\n- Scale the pixel values to fall within the [0,1] range using Min-Max Scaling formula.","metadata":{}},{"cell_type":"code","source":"# Method 1: Normalization to [0,1] using Min-Max Scaling\nx_train_min = np.min(x_train)\nx_train_max = np.max(x_train)\nx_train_normalized = (x_train - x_train_min) / (x_train_max - x_train_min)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:17.321025Z","iopub.execute_input":"2024-03-31T14:29:17.321343Z","iopub.status.idle":"2024-03-31T14:29:17.793671Z","shell.execute_reply.started":"2024-03-31T14:29:17.321314Z","shell.execute_reply":"2024-03-31T14:29:17.792766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test_normalized = (x_test - x_train_min) / (x_train_max - x_train_min)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:17.794929Z","iopub.execute_input":"2024-03-31T14:29:17.795236Z","iopub.status.idle":"2024-03-31T14:29:17.882569Z","shell.execute_reply.started":"2024-03-31T14:29:17.795210Z","shell.execute_reply":"2024-03-31T14:29:17.881738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 3.2 |</span> Normalization to [-1,1] (Standardisation)</b>","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Method 2: Normalization to [-1,1] following a Normal Distribution\nx_train_mean = np.mean(x_train)\nx_train_std = np.std(x_train)\nx_train_standardized = (x_train - x_train_mean) / x_train_std","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:17.883598Z","iopub.execute_input":"2024-03-31T14:29:17.883908Z","iopub.status.idle":"2024-03-31T14:29:19.212382Z","shell.execute_reply.started":"2024-03-31T14:29:17.883881Z","shell.execute_reply":"2024-03-31T14:29:19.211490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test_standardized = (x_test - x_train_mean) / x_train_std","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:19.213523Z","iopub.execute_input":"2024-03-31T14:29:19.213873Z","iopub.status.idle":"2024-03-31T14:29:19.322163Z","shell.execute_reply.started":"2024-03-31T14:29:19.213837Z","shell.execute_reply":"2024-03-31T14:29:19.320862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"padding: 30px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#FFCE30\"><b><span style='color:#FFFFFF'>4 |</span></b> <b>IMPLEMENTING A MULTI-LAYER PERCEPTRON FOR CIFAR-10 CLASSIFICATION</b></div>\n**Objective:** Build and train a Multi-Layer Perceptron (MLP) to classify images from the CIFAR-10 dataset, and achieve >50% accuracy.","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 4.1 |</span> Dataset Preparation</b>\n- Load and normalizae the CIFAR-10 dataset. \n- Flatten the images to use them with an MLP, as MLPs do not take the spatial structure of the images into account.\n- After flattening, each image is represented as a 1-dimensional array of size 3072 (32 * 32 * 3).\n- The shape attribute of the arrays provides information about the dimensions of the arrays.","metadata":{}},{"cell_type":"code","source":"# Flatten the images\nx_train_flat = x_train_normalized.reshape(x_train_normalized.shape[0], -1)\nx_test_flat = x_test_normalized.reshape(x_test_normalized.shape[0], -1)\n\n# Display the shape of flattened images\nprint(\"Flattened Training Images Shape:\", x_train_flat.shape)\nprint(\"Flattened Testing Images Shape:\", x_test_flat.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:19.323615Z","iopub.execute_input":"2024-03-31T14:29:19.323962Z","iopub.status.idle":"2024-03-31T14:29:20.224394Z","shell.execute_reply.started":"2024-03-31T14:29:19.323934Z","shell.execute_reply":"2024-03-31T14:29:20.223384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 4.2 |</span> Model Architecture</b>\n","metadata":{}},{"cell_type":"code","source":"# Define the MLP model\nmodel = tf.keras.models.Sequential(name=\"MLP_model\") \n\n# Add layers to the model\nmodel.add(tf.keras.layers.Dense(512, activation='relu', input_shape=(3072,)))  # 3072 = 32x32x3\nmodel.add(tf.keras.layers.Dense(10, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:20.225855Z","iopub.execute_input":"2024-03-31T14:29:20.226272Z","iopub.status.idle":"2024-03-31T14:29:20.842448Z","shell.execute_reply.started":"2024-03-31T14:29:20.226235Z","shell.execute_reply":"2024-03-31T14:29:20.841518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 4.3 |</span> Training</b>\n","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\n# Compile the model\noptimizer = Adam()  # Use Adam optimizer\nmodel.compile(optimizer=optimizer,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\n# Display the model summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:20.848267Z","iopub.execute_input":"2024-03-31T14:29:20.848587Z","iopub.status.idle":"2024-03-31T14:29:21.978165Z","shell.execute_reply.started":"2024-03-31T14:29:20.848563Z","shell.execute_reply":"2024-03-31T14:29:21.977195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model on the CIFAR-10 training set\nhistory = model.fit(x_train_flat, y_train,\n                    epochs=10,  \n                    batch_size=128,  \n                    validation_split=0.1)  # Use 10% of training data as validation\n\n# Evaluate the model on the test set\ntest_loss, test_accuracy = model.evaluate(x_test_flat, y_test)\nprint(\"Test Loss:\", test_loss)\nprint(\"Test Accuracy:\", test_accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:21.979467Z","iopub.execute_input":"2024-03-31T14:29:21.979743Z","iopub.status.idle":"2024-03-31T14:29:40.106948Z","shell.execute_reply.started":"2024-03-31T14:29:21.979718Z","shell.execute_reply":"2024-03-31T14:29:40.105958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 4.4 |</span> Evaluation</b>\n\n<span style=\"color: blue;\">\nThe model's performance on the CIFAR-10 test set is as follows:\n\n- Test Loss: 1.50\n- Test Accuracy: 0.4587 (45.87%)","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\nThe model's accuracy on the training set increases gradually over the epochs, reaching around 46% by the end of training.\nSimilarly, the validation accuracy also increases but remains slightly lower than the training accuracy, indicating that the model generalizes reasonably well to unseen data.\nThe model's loss (both training and validation) decreases consistently over the epochs, which is expected during the training process.\nHowever, there are still room for improvement as the accuracy is not > 50% yet.","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\nPossible next steps to improve:\n\n- Model Complexity: Experiment with increasing the model's complexity by adding more layers, neurons, or using different architectures such as convolutional neural networks (CNNs). CNNs are particularly effective for image classification tasks like CIFAR-10 as they can capture spatial patterns more effectively.\n\n- Hyperparameter Tuning: Experiment with different hyperparameters such as learning rate, batch size, number of epochs, and optimizer settings to find the optimal configuration for your model.\n\n- Data Augmentation: Augment the training data with techniques like rotation, scaling, flipping, or adding noise to increase the diversity of training samples and improve the model's robustness.","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 4.5 |</span> Hyperparameter Tuning</b>\n\nHyperparameter Tuning: Besides learning rate, batch size, and number of epochs, number and size of layers of MLP are also hyperparameters that can be tuned. Adding more layers and/or increasing the number of neurons per layer can enable the modeling of more complex functions, but this also increases the risk of overfitting and computational costs. It's advisable to start with simpler models and gradually increase complexity. Try the following architectures using ReLU as the activation function for hidden layers and softmax as the activation function for the output layer:\n\n- (1) MLP with one hidden layer consisting of 128 neurons. \n- (2) MLP with one hidden layer consisting of 256 neurons. \n- (3) MLP with two hidden layers consisting of 256 and 128 neurons, respectively.\n","metadata":{}},{"cell_type":"code","source":"# Define the MLP model architectures\ndef create_mlp_model_1():\n    model = tf.keras.models.Sequential([\n              \n        # Hidden layer with 128 neurons and ReLU activation\n        tf.keras.layers.Dense(128, activation='relu', input_shape=(3072,)), # 3072 = 32x32x3\n        \n        # Output layer with softmax activation for multi-class classification\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n    return model\n\ndef create_mlp_model_2():\n    model = tf.keras.models.Sequential([\n                \n        # Hidden layer with 256 neurons and ReLU activation\n        tf.keras.layers.Dense(256, activation='relu', input_shape=(3072,)), # 3072 = 32x32x3\n        \n        # Output layer with softmax activation for multi-class classification\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n    return model\n\ndef create_mlp_model_3():\n    model = tf.keras.models.Sequential([\n               \n        # First hidden layer with 256 neurons and ReLU activation\n        tf.keras.layers.Dense(256, activation='relu', input_shape=(3072,)), # 3072 = 32x32x3\n        \n        # Second hidden layer with 128 neurons and ReLU activation\n        tf.keras.layers.Dense(128, activation='relu'),\n        \n        # Output layer with softmax activation for multi-class classification\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:40.108224Z","iopub.execute_input":"2024-03-31T14:29:40.108523Z","iopub.status.idle":"2024-03-31T14:29:40.117256Z","shell.execute_reply.started":"2024-03-31T14:29:40.108497Z","shell.execute_reply":"2024-03-31T14:29:40.116182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile and train the model\ndef compile_and_train_model(model):\n    # Compile the model\n    optimizer = Adam()  # Use Adam optimizer\n    model.compile(optimizer=optimizer,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n    \n    # Train the model on the CIFAR-10 training set\n    history = model.fit(x_train_flat, y_train,\n                        epochs=10,  \n                        batch_size=128,  \n                        validation_split=0.1)  # Use 10% of training data as validation\n\n    # Evaluate the model on the test set\n    test_loss, test_accuracy = model.evaluate(x_test_flat, y_test)\n    print(\"Test Loss:\", test_loss)\n    print(\"Test Accuracy:\", test_accuracy)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:40.118637Z","iopub.execute_input":"2024-03-31T14:29:40.119025Z","iopub.status.idle":"2024-03-31T14:29:40.132437Z","shell.execute_reply.started":"2024-03-31T14:29:40.118994Z","shell.execute_reply":"2024-03-31T14:29:40.131574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Create and train models\nprint(\"MLP Model with one hidden layer (128 neurons):\")\nmlp_model_1 = create_mlp_model_1()\ncompile_and_train_model(mlp_model_1)\n\nprint(\"\\nMLP Model with one hidden layer (256 neurons):\")\nmlp_model_2 = create_mlp_model_2()\ncompile_and_train_model(mlp_model_2)\n\nprint(\"\\nMLP Model with two hidden layers (256 and 128 neurons):\")\nmlp_model_3 = create_mlp_model_3()\ncompile_and_train_model(mlp_model_3)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:29:40.133587Z","iopub.execute_input":"2024-03-31T14:29:40.133907Z","iopub.status.idle":"2024-03-31T14:30:26.876520Z","shell.execute_reply.started":"2024-03-31T14:29:40.133881Z","shell.execute_reply":"2024-03-31T14:30:26.875494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\nIt seems like model 3 (MLP with two hidden layers consisting of 256 and 128 neurons, respectively.) is the closest to 50% but not yet. The additional hidden layers seems to be giving better results. \n\nWe test below by: \n- adding one more layer with 512 neurons\n- adjust epoch to 20\n- adjust batch size to 256","metadata":{}},{"cell_type":"code","source":"# Define the MLP model\nmodel = tf.keras.models.Sequential(name=\"MLP_model_3layer\") \n\n# Add layers to the model\nmodel.add(tf.keras.layers.Dense(512, activation='relu', input_shape=(3072,)))  # 3072 = 32x32x3\nmodel.add(tf.keras.layers.Dense(256, activation='relu'))\nmodel.add(tf.keras.layers.Dense(128, activation='relu'))        \nmodel.add(tf.keras.layers.Dense(10, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:26.878084Z","iopub.execute_input":"2024-03-31T14:30:26.878392Z","iopub.status.idle":"2024-03-31T14:30:26.926418Z","shell.execute_reply.started":"2024-03-31T14:30:26.878365Z","shell.execute_reply":"2024-03-31T14:30:26.925452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\noptimizer = Adam()  # Use Adam optimizer\nmodel.compile(optimizer=optimizer,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:26.927520Z","iopub.execute_input":"2024-03-31T14:30:26.927827Z","iopub.status.idle":"2024-03-31T14:30:26.936086Z","shell.execute_reply.started":"2024-03-31T14:30:26.927789Z","shell.execute_reply":"2024-03-31T14:30:26.935074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model on the CIFAR-10 training set\nhistory = model.fit(x_train_flat, y_train,\n                    epochs=20,  \n                    batch_size=256,  \n                    validation_split=0.1)  # Use 10% of training data as validation\n\n# Evaluate the model on the test set\ntest_loss, test_accuracy = model.evaluate(x_test_flat, y_test)\nprint(\"Test Loss:\", test_loss)\nprint(\"Test Accuracy:\", test_accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:26.937476Z","iopub.execute_input":"2024-03-31T14:30:26.937868Z","iopub.status.idle":"2024-03-31T14:30:49.282181Z","shell.execute_reply.started":"2024-03-31T14:30:26.937835Z","shell.execute_reply":"2024-03-31T14:30:49.281235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\nThe combination of these changes led to improved performance and accuracy (>50%).\n\n- **Adding one more layer with 512 neurons at the start:** Adding more layers and neurons allows the model to capture more complex patterns and relationships within the data. With more neurons and layers, the model becomes more expressive and can learn more intricate representations of the input data. In essence, this change increases the capacity of the model to learn from the data.\n\n\n- **Adjusting epoch to 20:** Increasing the number of epochs allows the model to see the entire dataset more times during training. This extended training period provides more opportunities for the model to learn from the data and improve its performance. However, it's essential to be cautious about overfitting, which can occur if the model memorizes the training data rather than learning generalizable patterns.\n\n\n- **Adjusting batch size to 256:** Batch size determines the number of samples used in each iteration of training. A larger batch size can lead to faster convergence during training as the model updates its parameters less frequently. It can also provide more stable gradients, which can lead to better generalization. However, larger batch sizes may require more memory and computational resources.\n\n","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 4.6 |</span> Visually checking the MLP model predictions (51.8% Accuracy)</b>","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Get predictions for the test set\npredictions_mlp = model.predict(x_test_flat)\npredicted_labels_mlp = np.argmax(predictions_mlp, axis=1)\n\n# Plot sample images with predicted labels\nplt.figure(figsize=(10, 10))\nfor i in range(25):  # Adjust as needed\n    plt.subplot(5, 5, i + 1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(x_test[i])  # Plot original image\n    true_label = class_names[y_test[i][0]]  # Get true label\n    predicted_label_mlp = class_names[predicted_labels_mlp[i]]  # Get predicted label\n    if true_label == predicted_label_mlp:\n        color = 'green'  # Correct prediction\n    else:\n        color = 'red'  # Incorrect prediction\n    plt.xlabel(f\"True: {true_label}\\nPred (MLP): {predicted_label_mlp}\", color=color)\nplt.tight_layout()  # Adjust subplot layout to prevent overlap\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:49.283651Z","iopub.execute_input":"2024-03-31T14:30:49.284136Z","iopub.status.idle":"2024-03-31T14:30:52.213184Z","shell.execute_reply.started":"2024-03-31T14:30:49.284100Z","shell.execute_reply":"2024-03-31T14:30:52.212180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"padding: 30px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#FFCE30\"><b><span style='color:#FFFFFF'>5 |</span></b> <b>BUILDING A CONVOLUTIONAL NEURAL NETWORK FOR CIFAR-10</b></div>\n**Objective:** Develop a Convolutional Neural Network (CNN) to improve the classification performance on the CIFAR-10 dataset, and achieve >70% accuracy.","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 5.1 |</span> Dataset Preparation</b>\n- Load and normalize the CIFAR-10 dataset. \n- Ensure the images are normalized to a range suitable for CNN processing.","metadata":{}},{"cell_type":"code","source":"# Convert to float32\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\n\n# Normalize the training set\nx_train_min = x_train.min(axis=(0, 1, 2), keepdims=True)\nx_train_max = x_train.max(axis=(0, 1, 2), keepdims=True)\nx_train_normalized = (x_train - x_train_min) / (x_train_max - x_train_min)\n\n# Normalize the test set using training set statistics\nx_test_normalized = (x_test - x_train_min) / (x_train_max - x_train_min)\n\n# Display the shapes of the normalized datasets\nprint(\"x_train_normalized shape:\", x_train_normalized.shape)\nprint(\"x_test_normalized shape:\", x_test_normalized.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:52.214388Z","iopub.execute_input":"2024-03-31T14:30:52.214697Z","iopub.status.idle":"2024-03-31T14:30:53.102316Z","shell.execute_reply.started":"2024-03-31T14:30:52.214671Z","shell.execute_reply":"2024-03-31T14:30:53.101334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 5.2 |</span> CNN Architecture</b>\n- Design a CNN that includes convolutional layers, activation functions, pooling layers, and fully connected layers. \n- Detail your choice of kernel sizes, pooling sizes, and the architecture's depth.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers, models\n\n# Define the CNN model\nmodel = models.Sequential(name=\"CNN_model\")\n\n# Convolutional layers\nmodel.add(layers.Conv2D(32, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu', padding='same'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu', padding='same'))\nmodel.add(layers.MaxPooling2D((2, 2)))\n\n# Flatten layer to transition from convolutional to dense layers\nmodel.add(layers.Flatten())\n\n# Dense (fully connected) layers\nmodel.add(layers.Dense(128, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))  # Output layer with 10 classes\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:53.103584Z","iopub.execute_input":"2024-03-31T14:30:53.104401Z","iopub.status.idle":"2024-03-31T14:30:53.188062Z","shell.execute_reply.started":"2024-03-31T14:30:53.104364Z","shell.execute_reply":"2024-03-31T14:30:53.187145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\n- The first layer has 32 filters, followed by a ReLU activation function, 'same' padding, and an input shape of (32, 32, 3) which corresponds to 32x32 RGB images. \n- The subsequent layers continue to add convolutional layers with different numbers of filters and ReLU activations. \n- MaxPooling layers with a pool size of (2, 2) are added after the convolutional layers to downsample the feature maps.\n- A Flatten layer is added to transition from the convolutional layers to the dense layers. It flattens the input without affecting the batch size.\n- We then add dense (fully connected) layers to the model. The first layer has 128 neurons with ReLU activation, and the output layer has 10 neurons (one for each class in CIFAR-10) with softmax activation, which is typical for multi-class classification problems.","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 5.3 |</span> Training</b>\n- Compile your CNN with a suitable loss function and optimizer. \n- Utilize techniques such as dropout and batch normalization to prevent overfitting and ensure more stable training.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers, models\n\n# Define the CNN model\nmodel = models.Sequential(name=\"CNN_model\")\n\n# Convolutional layers\nmodel.add(layers.Conv2D(32, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu', padding='same'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Dropout(0.25))  # Dropout layer to prevent overfitting\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu', padding='same'))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Dropout(0.25))  # Dropout layer\n\n# Flatten layer to transition from convolutional to dense layers\nmodel.add(layers.Flatten())\n\n# Dense (fully connected) layers with batch normalization and dropout\nmodel.add(layers.Dense(128, activation='relu'))\nmodel.add(layers.BatchNormalization())  # Batch normalization layer\nmodel.add(layers.Dropout(0.5))  # Dropout layer\nmodel.add(layers.Dense(10, activation='softmax'))  # Output layer with 10 classes\n\n# Compile the model with suitable loss function and optimizer\noptimizer = Adam(learning_rate=0.0001)  # Use Adam optimizer\nmodel.compile(optimizer=optimizer,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\n# Display the model summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:53.189290Z","iopub.execute_input":"2024-03-31T14:30:53.189632Z","iopub.status.idle":"2024-03-31T14:30:53.616295Z","shell.execute_reply.started":"2024-03-31T14:30:53.189601Z","shell.execute_reply":"2024-03-31T14:30:53.615339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 5.4 |</span> Evaluation and Comparison</b>\n- Report the accuracy of the model on the CIFAR-10 test set. \n- Compare the performance of your CNN to the MLP model from Question 2, discussing why the CNN performs differently.","metadata":{}},{"cell_type":"code","source":"# Fit the CNN model on the training set\nhistory = model.fit(x_train_normalized, y_train,\n                    epochs=10,  \n                    batch_size=128,  \n                    validation_split=0.1)  # Use 10% of training data as validation\n\n# Evaluate the CNN model on the test set\ntest_loss, test_accuracy = model.evaluate(x_test_normalized, y_test)\nprint(\"Test Loss:\", test_loss)\nprint(\"Test Accuracy:\", test_accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:30:53.617530Z","iopub.execute_input":"2024-03-31T14:30:53.617854Z","iopub.status.idle":"2024-03-31T14:31:48.981019Z","shell.execute_reply.started":"2024-03-31T14:30:53.617820Z","shell.execute_reply":"2024-03-31T14:31:48.979996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\nComparing the performance of the CNN to the MLP model from Question 2, this are potential reasons for differences in performance:\n\n- **CNN's Ability to Capture Spatial Patterns:** CNNs are specifically designed to capture spatial patterns in images. They use convolutional layers that scan the input image with learnable filters to detect local patterns. This ability to capture spatial information can lead to better performance on image classification tasks like CIFAR-10 compared to MLPs, which treat images as flattened vectors and do not consider spatial relationships.\n\n\n- **Parameter Efficiency:** CNNs typically have fewer parameters compared to fully connected MLPs. CNNs share weights across spatial locations, which allows them to learn efficiently from limited training data. On the other hand, MLPs have fully connected layers, which can lead to a larger number of parameters and increased risk of overfitting, especially on small datasets like CIFAR-10.\n\n\n- **Feature Hierarchies:** CNNs learn hierarchical representations of features from raw data. Convolutional layers extract low-level features like edges and textures, while deeper layers combine these features to learn more abstract representations. This hierarchical feature learning enables CNNs to effectively represent complex patterns in images, which can result in better performance on image classification tasks.\n\n\n- **Regularization Techniques:** In the CNN architecture provided, dropout and batch normalization layers are used to prevent overfitting and stabilize training. These regularization techniques help improve the generalization ability of the CNN model by reducing the risk of overfitting to the training data.","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 5.5 |</span> Visually checking the model predictions (63.5% Accuracy)</b>","metadata":{}},{"cell_type":"code","source":"# Get predictions for the test set\npredictions_cnn = model.predict(x_test_normalized)\npredicted_labels_cnn = np.argmax(predictions_cnn, axis=1)\n\n# Plot sample images with predicted labels\nplt.figure(figsize=(10, 10))\nfor i in range(25):  # Adjust as needed\n    plt.subplot(5, 5, i + 1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(x_test_normalized[i])  # Plot original image\n    true_label = class_names[y_test[i][0]]  # Get true label\n    predicted_label = class_names[predicted_labels_cnn[i]]  # Get predicted label\n    if true_label == predicted_label:\n        color = 'green'  # Correct prediction\n    else:\n        color = 'red'  # Incorrect prediction\n    plt.xlabel(f\"True: {true_label}\\nPred (CNN): {predicted_label}\", color=color)\nplt.tight_layout()  # Adjust subplot layout to prevent overlap\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:48.982351Z","iopub.execute_input":"2024-03-31T14:31:48.982639Z","iopub.status.idle":"2024-03-31T14:31:52.048293Z","shell.execute_reply.started":"2024-03-31T14:31:48.982614Z","shell.execute_reply":"2024-03-31T14:31:52.047361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"padding: 30px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#FFCE30\"><b><span style='color:#FFFFFF'>6 |</span></b> <b>LEVRAGING A PRETRAINED MODEL FOR CIFAR-10</b></div>\n**Objective:** Use a pretrained model from TensorFlow's Keras applications or PyTorch's torchvision models as a feature extractor or for fine-tuning on the CIFAR-10 dataset, and achieve >90% accuracy on the test dataset of CIFAR-10. .","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 6.1 |</span> Dataset Preparation</b>\n- Load the CIFAR-10 dataset, applying normalization and any required preprocessing to match the input format of the pretrained model.","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n    \n- The CIFAR-10 dataset is loaded using the tf.keras.datasets.cifar10.load_data() function.\n- After loading, the input images are converted to float32 data type and normalized.\n- Preprocessing is applied using the ResNet50 preprocessing function tf.keras.applications.resnet50.preprocess_input().\n- Train test split is used to split out train set at 0.1 which we will need to use for validation later (with data augmentation)\n- There is also a OHE performed in order to better allow the classess to be distinct\n\n</span>","metadata":{}},{"cell_type":"code","source":"# Load CIFAR-10 dataset\n(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:52.049489Z","iopub.execute_input":"2024-03-31T14:31:52.049787Z","iopub.status.idle":"2024-03-31T14:31:52.881495Z","shell.execute_reply.started":"2024-03-31T14:31:52.049761Z","shell.execute_reply":"2024-03-31T14:31:52.880582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Preprocess input images\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\n\n# Preprocess input images using ResNet50 preprocessing\nx_train = tf.keras.applications.resnet50.preprocess_input(x_train)\nx_test = tf.keras.applications.resnet50.preprocess_input(x_test)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:52.882671Z","iopub.execute_input":"2024-03-31T14:31:52.882998Z","iopub.status.idle":"2024-03-31T14:31:53.413453Z","shell.execute_reply.started":"2024-03-31T14:31:52.882970Z","shell.execute_reply":"2024-03-31T14:31:53.412448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train,x_val,y_train,y_val=train_test_split(x_train,y_train,test_size=.1)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:53.414538Z","iopub.execute_input":"2024-03-31T14:31:53.414841Z","iopub.status.idle":"2024-03-31T14:31:53.612034Z","shell.execute_reply.started":"2024-03-31T14:31:53.414795Z","shell.execute_reply":"2024-03-31T14:31:53.611220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import to_categorical\n\n#One hot encode the labels.Since we have 10 classes we should expect the shape[1] of y_train,y_val and y_test to change from 1 to 10\n\ny_train=to_categorical(y_train)\ny_val=to_categorical(y_val)\ny_test=to_categorical(y_test)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:53.613072Z","iopub.execute_input":"2024-03-31T14:31:53.613362Z","iopub.status.idle":"2024-03-31T14:31:53.620299Z","shell.execute_reply.started":"2024-03-31T14:31:53.613337Z","shell.execute_reply":"2024-03-31T14:31:53.619471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Print the dimensions of the datasets to check\n\nprint((x_train.shape,y_train.shape))\nprint((x_val.shape,y_val.shape))\nprint((x_test.shape,y_test.shape))","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:53.621425Z","iopub.execute_input":"2024-03-31T14:31:53.621767Z","iopub.status.idle":"2024-03-31T14:31:53.635185Z","shell.execute_reply.started":"2024-03-31T14:31:53.621734Z","shell.execute_reply":"2024-03-31T14:31:53.634346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n#Data Augmentation Function: Let's define an instance of the ImageDataGenerator class and set the parameters.\ntrain_generator = ImageDataGenerator(\n    rotation_range=10,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    horizontal_flip=True,\n    shear_range=0.1,\n    zoom_range=0.1,)\n\nval_generator = ImageDataGenerator(\n    rotation_range=10,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    horizontal_flip=True,\n    shear_range=0.1,\n    zoom_range=0.1,)\n\ntest_generator = ImageDataGenerator(\n    rotation_range=10,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    horizontal_flip=True,\n    shear_range=0.1,\n    zoom_range=0.1,) ","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:53.636387Z","iopub.execute_input":"2024-03-31T14:31:53.636834Z","iopub.status.idle":"2024-03-31T14:31:53.648117Z","shell.execute_reply.started":"2024-03-31T14:31:53.636781Z","shell.execute_reply":"2024-03-31T14:31:53.647255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 6.2 |</span> Model Selection, Modification and Training Strategy</b>\n- Choose a pretrained model (e.g., ResNet, VGG16) and modify it for CIFAR-10 classification. \n- Describe how you adapt the model for the 10 classes of CIFAR-10 (e.g., modifying the top layer, adjusting input size).\n- Decide whether to freeze the weights of the pretrained layers and only train the top layer(s), or to fine-tune the entire network. \n- Justify your choice based on the CIFAR-10 dataset's characteristics.","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\n**Chosen Model and exclude top layer**\n- The chosen pretrained model is ResNet50, which is loaded using the tf.keras.applications.ResNet50 function. \n- The model is configured to exclude the top classification layer (include_top=False).\n\n</span>","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\n**Adjusting inputsize and modify top layer**\n\n- The input images for CIFAR-10 are 32x32 pixels, while ResNet50 expects input images of size 224x224 pixels.\n- Therefore, the input images are upsampled to match the input size required by ResNet50.\r\n-\r\nThe classification layers are modified by adding a global average pooling layer and two dense lay\n- . The final dense layer has 10 units with softmax activation, corresponding to the 10 class in CIFAR-10.","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\n**Unfreezing some pre-trained layers**\n- In this case, some of the weights of the pretrained layers are not frozen. As such, they will be fine-tuned during training. \n- This decision allows the model to adapt slightly to the features specific to the CIFAR-10 dataset.\n- The down-side is that it could take more parameters and therefore more computational time\n\n</span>","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\n**Justification of choices**\n- CIFAR-10 is a relatively small dataset with low-resolution images.\n- Fine-tuning the some of the network allows the model to learn more specific features relevant to the CIFAR-10 dataset.\n- Since ResNet50 is pretrained on a large dataset (ImageNet) and has already learned valuable hierarchical features, fine-tuning some layers in the entire network can help improve model performance on CIFAR-10.\n- To set it as a variable to decide how many layers to unfreeze","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.applications import ResNet50  # Import ResNet50 model\nfrom tensorflow.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Dropout, BatchNormalization, UpSampling2D\nfrom tensorflow.keras.models import Model\n\n# Feature Extraction is performed by ResNet50 pretrained on imagenet weights. \n# Input size is 224 x 224.\ninputs = tf.keras.layers.Input(shape=(32, 32, 3))\n\n# Upsample the input image to match the size expected by ResNet50\nresized_inputs = UpSampling2D(size=(7, 7))(inputs)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:53.649277Z","iopub.execute_input":"2024-03-31T14:31:53.649883Z","iopub.status.idle":"2024-03-31T14:31:53.665326Z","shell.execute_reply.started":"2024-03-31T14:31:53.649853Z","shell.execute_reply":"2024-03-31T14:31:53.664394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the ResNet50 model with pretrained weights\nresnet_model = ResNet50(\n    include_top=False,\n    weights='imagenet',\n    input_tensor=resized_inputs  # Use resized input as input tensor\n)\n\n# Freeze all layers initially\nfor layer in resnet_model.layers:\n    layer.trainable = False\n\n# Unfreeze the last X layers of the ResNet50 model\nNUM_LAYERS_TO_UNFREEZE = 30\nfor layer in resnet_model.layers[-NUM_LAYERS_TO_UNFREEZE:]:\n    layer.trainable = True\n\n# Global average pooling and classification layers\nx = GlobalAveragePooling2D()(resnet_model.output)\nx = Dense(1024, activation=\"relu\")(x)\nx = BatchNormalization()(x)  \nx = Dropout(0.3)(x)  \nx = Dense(512, activation=\"relu\")(x)\nx = BatchNormalization()(x)  \nx = Dropout(0.2)(x)  \nx = Dense(256, activation=\"relu\")(x)\nx = BatchNormalization()(x)  \nx = Dropout(0.2)(x)  \nclassification_output = Dense(10, activation=\"softmax\", name=\"classification\")(x)\n\n# Connect the feature extraction and \"classifier\" layers to build the model\nResNet_model = Model(inputs=inputs, outputs=classification_output, name=\"ResNet\")\n\n# Compile the model\noptimizer = tf.keras.optimizers.Adam()  # Use Adam optimizer\n\n\n#Specify the loss as categorical_crossentropy since the labels are 1 hot encoded. \n#If labels are integer, use sparse categorical crossentropy as loss function.\nResNet_model.compile(optimizer=optimizer,\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\nResNet_model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:53.666530Z","iopub.execute_input":"2024-03-31T14:31:53.666871Z","iopub.status.idle":"2024-03-31T14:31:55.918369Z","shell.execute_reply.started":"2024-03-31T14:31:53.666846Z","shell.execute_reply":"2024-03-31T14:31:55.917433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of layers\nnum_layers = len(ResNet_model.layers)\nprint(\"Number of layers in the model:\", num_layers)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:55.919394Z","iopub.execute_input":"2024-03-31T14:31:55.919652Z","iopub.status.idle":"2024-03-31T14:31:55.925054Z","shell.execute_reply.started":"2024-03-31T14:31:55.919629Z","shell.execute_reply":"2024-03-31T14:31:55.923963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 6.3 |</span> Evaluation</b>\n- Test the model on the CIFAR-10 test set and report the accuracy. \n- Compare the results with the MLP and CNN models from the previous questions, discussing the advantages and limitations of using pretrained models on a dataset like CIFAR-10.","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\n# Add ReduceLROnPlateau callback\n# Here, the learning rate will be reduced by half (factor=0.5) if no improvement in validation loss is observed for 1 epochs\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.00001)\n\n# Add EarlyStopping callback\n# Here, training will be stopped if no improvement in validation loss is observed for 5 epochs.\n# The `restore_best_weights` parameter ensures that the model weights are reset to the values from the epoch\n# with the best value of the monitored quantity (in this case, 'val_loss').\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True, verbose=1)\n\n\nbatch_size=64\nepochs=15\n\nhistory = ResNet_model.fit(train_generator.flow(x_train,y_train,batch_size=batch_size),\n                                     epochs=epochs,\n                                     steps_per_epoch=x_train.shape[0]//batch_size,\n                                     validation_data=val_generator.flow(x_val,y_val,batch_size=batch_size),\n                                     #validation_steps=250,\n                                     callbacks=[reduce_lr, early_stopping],\n                                     shuffle = True\n                                    )\n# Evaluate the model\ntest_loss, test_accuracy = ResNet_model.evaluate(test_generator.flow(x_test,y_test,batch_size=batch_size))\nprint(\"Test Loss:\", test_loss)\nprint(\"Test Accuracy:\", test_accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:31:55.926183Z","iopub.execute_input":"2024-03-31T14:31:55.926449Z","iopub.status.idle":"2024-03-31T14:54:45.268615Z","shell.execute_reply.started":"2024-03-31T14:31:55.926426Z","shell.execute_reply":"2024-03-31T14:54:45.267475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\nAfter multiple iterations, combinations as well as computational time, we finally arrive at a test accuracy of 0.914 that is > 90%.\nThe set-up of the model to achieve this is as such:\n\n- Train test split for validation data\n- OHE the categories of classes\n- Use image generator for Data augmentation\n- Use ResNet50 and exclude top layer and add our own Nueral Network to Batch Normalise and droupout before classifying into 10 nodes\n- Used upsampling (7,7) to match with CIFAR-10 (32 x 32) images with ResNet 224 x 224 input size\n- Initially freeze all layers but subsuequently unfreeze 20 layers to allow some weights in ResNet50 to be updated and improve performance\n- Use Adam optimizer rather than SGD as if has adaptive learning rates\n- Include reduce learning rate and early stopping functions in fitting to ensure each epoch helps to improve the result\n- include shuffle = True to ensure model does not learn also the sequence of data and cause overfitting","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: blue;\">\n\n**Advantages of using pretrained models like ResNet on CIFAR-10:**\n\n- Transfer Learning: Pretrained models come with knowledge learned from large-scale datasets like ImageNet, which can be transferred to CIFAR-10, improving performance with limited labeled data.\n- Feature Extraction: Pretrained models can extract meaningful features from images, capturing hierarchical representations that might not be effectively learned from scratch.\n- Higher accuracy: Using a pretrained model that is based on a much larger dataset can help to achieve much higher accuracy than other models built from scratch since the training data was much larger than what we have.\n\n**Limitations of using pretrained models:**\n\n- Domain Adaptation: Pretrained models might not perfectly adapt to the characteristics of CIFAR-10, leading to suboptimal performance.\n- Task Specificity: Pretrained models are designed for specific tasks, and their features might not always be relevant to CIFAR-10 classification tasks.\n- Overfitting Risk: Fine-tuning the entire pretrained model on a small dataset like CIFAR-10 can lead to overfitting if not properly regularized.","metadata":{}},{"cell_type":"markdown","source":"### <b><span style='color:#FFCE30'> 6.4 |</span> Visually checking the model predictions (91.4% Accuracy)</b>","metadata":{}},{"cell_type":"code","source":"# Get predictions for the test set\npredictions_resnet = ResNet_model.predict(x_test)\npredicted_labels_resnet = np.argmax(predictions_resnet, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:54:45.270085Z","iopub.execute_input":"2024-03-31T14:54:45.270446Z","iopub.status.idle":"2024-03-31T14:55:23.225862Z","shell.execute_reply.started":"2024-03-31T14:54:45.270416Z","shell.execute_reply":"2024-03-31T14:55:23.224799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot sample images with predicted labels\nplt.figure(figsize=(10, 10))\nfor i in range(25):  # Adjust as needed\n    plt.subplot(5, 5, i + 1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    # Revert preprocessing for visualization by reloading CIFAR-10 dataset\n    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()\n    plt.imshow(x_test[i])  #Plot original picture\n    true_label = class_names[y_test[i][0]]  # Get true label\n    predicted_label = class_names[predicted_labels_resnet[i]]  # Get predicted label\n    if true_label == predicted_label:\n        color = 'green'  # Correct prediction\n    else:\n        color = 'red'  # Incorrect prediction\n    plt.xlabel(f\"True: {true_label}\\nPred (ResNet): {predicted_label}\", color=color)\nplt.tight_layout()  # Adjust subplot layout to prevent overlap\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-31T14:55:23.227284Z","iopub.execute_input":"2024-03-31T14:55:23.227611Z","iopub.status.idle":"2024-03-31T14:55:45.789220Z","shell.execute_reply.started":"2024-03-31T14:55:23.227581Z","shell.execute_reply":"2024-03-31T14:55:45.788230Z"},"trusted":true},"execution_count":null,"outputs":[]}]}