{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom time import time as timer\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nfrom torch.utils.data import random_split, Dataset, DataLoader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:35.733656Z","iopub.execute_input":"2025-03-05T15:36:35.733963Z","iopub.status.idle":"2025-03-05T15:36:35.738520Z","shell.execute_reply.started":"2025-03-05T15:36:35.733938Z","shell.execute_reply":"2025-03-05T15:36:35.737796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:38.107848Z","iopub.execute_input":"2025-03-05T15:36:38.108127Z","iopub.status.idle":"2025-03-05T15:36:38.116063Z","shell.execute_reply.started":"2025-03-05T15:36:38.108107Z","shell.execute_reply":"2025-03-05T15:36:38.115341Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Looking at Sample Submissions, and labels CSV","metadata":{}},{"cell_type":"code","source":"labels_csv = pd.read_csv(\"/kaggle/input/cifar-10/trainLabels.csv\")\nlabels_csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:40.168115Z","iopub.execute_input":"2025-03-05T15:36:40.168446Z","iopub.status.idle":"2025-03-05T15:36:40.189446Z","shell.execute_reply.started":"2025-03-05T15:36:40.168421Z","shell.execute_reply":"2025-03-05T15:36:40.188739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels_csv.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:40.280372Z","iopub.execute_input":"2025-03-05T15:36:40.280602Z","iopub.status.idle":"2025-03-05T15:36:40.291237Z","shell.execute_reply.started":"2025-03-05T15:36:40.280583Z","shell.execute_reply":"2025-03-05T15:36:40.290517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Since the labels are stored as text in the DataFrame, we'll have to the map the labels to numbers\nlabel_mapping = {label: idx for idx, label in enumerate(labels_csv['label'].unique())}\n\n# Now let's encode them, and rename the text column\nlabels_csv.rename({\"label\": \"label_txt\"}, axis=1, inplace=True)\nlabels_csv['label'] = labels_csv['label_txt'].map(label_mapping)\n\nlabel_mapping, labels_csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:40.397016Z","iopub.execute_input":"2025-03-05T15:36:40.397261Z","iopub.status.idle":"2025-03-05T15:36:40.410700Z","shell.execute_reply.started":"2025-03-05T15:36:40.397242Z","shell.execute_reply":"2025-03-05T15:36:40.409921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submissions_csv = pd.read_csv(\"/kaggle/input/cifar-10/sampleSubmission.csv\")\nsample_submissions_csv.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:40.523796Z","iopub.execute_input":"2025-03-05T15:36:40.524018Z","iopub.status.idle":"2025-03-05T15:36:40.594033Z","shell.execute_reply.started":"2025-03-05T15:36:40.524001Z","shell.execute_reply":"2025-03-05T15:36:40.593367Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extracting the training images from the 7z folder","metadata":{}},{"cell_type":"code","source":"# To extract the train and test images\n# We'll extract only half right now, then break the execution\n!sudo apt-get install p7zip-full\n!7z x \"/kaggle/input/cifar-10/train.7z\"\n# !7z x \"/kaggle/input/cifar-10/test.7z\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:40.749638Z","iopub.execute_input":"2025-03-05T15:36:40.749896Z","iopub.status.idle":"2025-03-05T15:36:49.143999Z","shell.execute_reply.started":"2025-03-05T15:36:40.749875Z","shell.execute_reply":"2025-03-05T15:36:49.143117Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Checking the data","metadata":{}},{"cell_type":"code","source":"train_dir = \"/kaggle/working/train\"\n# test_dir = \"/kaggle/working/test\"\n\ntrain_imgs = os.listdir(train_dir)\n\nprint(f\"Number of train images: {len(train_imgs)}\")\nprint(f\"First 5 images' paths: {train_imgs[:5]}\")\n\nimg1 = Image.open(train_dir + \"/\" + train_imgs[0])\nimg1.size","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:36:58.839894Z","iopub.execute_input":"2025-03-05T15:36:58.840239Z","iopub.status.idle":"2025-03-05T15:36:58.873441Z","shell.execute_reply.started":"2025-03-05T15:36:58.840216Z","shell.execute_reply":"2025-03-05T15:36:58.872744Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Applying Image Transforms (For Sample Image)","metadata":{}},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))    # To shift each pixel's value between -1 and 1, helping avoid exploding and vanishing gradients\n])\n\nimg_tensor = transform(img1)\nimg_tensor.shape   # Shape is 32x32x3, which tracks with how it should be","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:01.329049Z","iopub.execute_input":"2025-03-05T15:37:01.329382Z","iopub.status.idle":"2025-03-05T15:37:01.336363Z","shell.execute_reply.started":"2025-03-05T15:37:01.329355Z","shell.execute_reply":"2025-03-05T15:37:01.335607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Applying Train Image Transforms (utilizing Data Augmentation), and Test Image Transforms","metadata":{}},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(p=0.2),   # Randomly flip about 20% images horizontally, to help model learn right-left symmetry\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,)),   # To shift each pixel's value between [-1, 1], helping avoid exploding and vanishing gradients\n])\n\n# RandomCrop not applied, because Cropping an image in the CIFAR-10 dataset may make the image inconclusive.\n\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:04.045649Z","iopub.execute_input":"2025-03-05T15:37:04.045940Z","iopub.status.idle":"2025-03-05T15:37:04.051447Z","shell.execute_reply.started":"2025-03-05T15:37:04.045918Z","shell.execute_reply":"2025-03-05T15:37:04.050526Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Loading Data to DataLoaders","metadata":{}},{"cell_type":"markdown","source":"#### Creating a Custom Dataset class in order to load the images as a torch.Dataset","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, images_dir, labels_csv, transform=None):\n        self.transforms = transform\n        self.imgs_dir = images_dir\n        self.labels_csv = labels_csv\n\n    def __len__(self):\n        return len(self.labels_csv)\n    \n    # Function to load each image, since we don't have the default structure that PyTorch wants it's Datasets to have\n    def __getitem__(self, idx):\n        img_name = str(self.labels_csv.iloc[idx, 0]) + \".png\"    # Get file name of image with index \"idx\"\n        label = self.labels_csv.loc[idx, \"label\"]   # Get label of the image with index \"idx\"\n\n        img_path = os.path.join(self.imgs_dir, img_name)\n\n        img = Image.open(img_path).convert(\"RGB\")   # Open the image in a PIL.Image format, and convert it to an RGB image\n\n        # Applying transforms\n        if self.transforms:\n            img = self.transforms(img)\n        \n        return img, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:10.485777Z","iopub.execute_input":"2025-03-05T15:37:10.486085Z","iopub.status.idle":"2025-03-05T15:37:10.491133Z","shell.execute_reply.started":"2025-03-05T15:37:10.486060Z","shell.execute_reply":"2025-03-05T15:37:10.490349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating the DataLoader for training data\ntrain_dataset = CustomDataset(\n    images_dir = \"/kaggle/working/train\", labels_csv = labels_csv,\n    transform = transform\n)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:12.613579Z","iopub.execute_input":"2025-03-05T15:37:12.613879Z","iopub.status.idle":"2025-03-05T15:37:12.618576Z","shell.execute_reply.started":"2025-03-05T15:37:12.613858Z","shell.execute_reply":"2025-03-05T15:37:12.617527Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Creating a CNN Architecture\n\nFor the purpose of this problem, we'll be using the following architecture:\n\n* Input [ Dimensions: (Batch_size (32), 3, 32, 32) ]\n* First Convolution Layer [ (32, 3, 32, 32) -> (32, 32, 32, 32) ]\n* Second Convolution Layer [ (32, 32, 32, 32) -> (32, 64, 32, 32) ]\n* First Max Pooling Layer [ (32, 64, 32, 32) -> (32, 64, 16, 16) ]\n* Third Convolution Layer [ (32, 64, 16, 16) -> (32, 128, 16, 16) ]\n* Second Max Pooling Layer [ (32, 128, 16, 16) -> (32, 128, 8, 8) ]\n* Flatten Layer [ (32, 128, 8, 8) -> (32, 8192) ]\n* First FC Layer [ (32, 8192) -> (32, 256) ]\n* Second FC Layer [ (32, 256) -> (32, 128) ]\n* Dropout with keep_probability = 0.3\n* Output Layer [ (32, 128) -> (32, 10) ]","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        self.conv_layers = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1),  # Conv Layer 1\n            nn.ReLU(),\n            nn.Conv2d(32, 64, kernel_size=3, padding=1),  # Conv Layer 2\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),  # Max Pooling 1\n            \n            nn.Conv2d(64, 128, kernel_size=3, padding=1),  # Conv Layer 3\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)  # Max Pooling 2\n        )\n        \n        self.fc_layers = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 8 * 8, 256),  # Fully connected layer 1\n            nn.ReLU(),\n            nn.Linear(256, 128),  # Fully connected layer 2\n            nn.ReLU(),\n            nn.Dropout(0.3),  # Dropout for regularization\n            nn.Linear(128, 10)  # Output layer (10 classes)\n        )\n    \n    def forward(self, x):\n        x = self.conv_layers(x)\n        x = self.fc_layers(x)\n        return x\n\n# Initialize model\nmodel = CNN()\nprint(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:20.210764Z","iopub.execute_input":"2025-03-05T15:37:20.211045Z","iopub.status.idle":"2025-03-05T15:37:20.241443Z","shell.execute_reply.started":"2025-03-05T15:37:20.211025Z","shell.execute_reply":"2025-03-05T15:37:20.240783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Shapes of all parameters\nfor param in model.parameters():\n    print(param.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:23.230348Z","iopub.execute_input":"2025-03-05T15:37:23.230635Z","iopub.status.idle":"2025-03-05T15:37:23.236199Z","shell.execute_reply.started":"2025-03-05T15:37:23.230613Z","shell.execute_reply":"2025-03-05T15:37:23.235427Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For this particular problem, we'll be using the Cross Entropy loss function, and the Adam optimizer so we'll go ahead and set that up, before we make the training loop","metadata":{}},{"cell_type":"code","source":"# Setting the Loss criteria and the optimizer\nloss = nn.CrossEntropyLoss()   # Not MSE as this is not a regression problem, but a multi classification problem\noptimizer = optim.Adam(model.parameters(), lr=1e-3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:27.899926Z","iopub.execute_input":"2025-03-05T15:37:27.900286Z","iopub.status.idle":"2025-03-05T15:37:27.904882Z","shell.execute_reply.started":"2025-03-05T15:37:27.900256Z","shell.execute_reply":"2025-03-05T15:37:27.903877Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Migrating model to GPU, if available","metadata":{}},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device = torch.device(\"cuda\")\nelse:\n    device = torch.device(\"cpu\")\n\ndevice","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:29.975584Z","iopub.execute_input":"2025-03-05T15:37:29.975939Z","iopub.status.idle":"2025-03-05T15:37:29.981504Z","shell.execute_reply.started":"2025-03-05T15:37:29.975910Z","shell.execute_reply":"2025-03-05T15:37:29.980580Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Creating a simple training loop","metadata":{}},{"cell_type":"code","source":"def train(train_loader, model, loss_fn, optimizer, device, n_epochs=50):\n    model.to(device)\n\n    for epoch_num in range(1, n_epochs+1):\n        start_time = time.time()\n        model.train()    # Set the model to training model\n        running_loss = 0.0\n        \n        for images, labels in train_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n    \n            optimizer.zero_grad()            # Reset gradients to 0\n    \n            outputs = model(images)           # Forward pass\n            loss = loss_fn(outputs, labels)\n            loss.backward()                 # Start Backprop\n            \n            optimizer.step()                 # Update gradients\n    \n            running_loss += loss.item()\n    \n        epoch_loss = running_loss / len(train_loader)\n\n        print(f\"Epoch {epoch_num}/{n_epochs}; Loss = {epoch_loss:.5f}; Time: {time.time()-start_time}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:36.363218Z","iopub.execute_input":"2025-03-05T15:37:36.363549Z","iopub.status.idle":"2025-03-05T15:37:36.368870Z","shell.execute_reply.started":"2025-03-05T15:37:36.363523Z","shell.execute_reply":"2025-03-05T15:37:36.368016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sample checking if training works\ntrain(train_loader, model, loss, optimizer, device, 3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:15:26.936653Z","iopub.execute_input":"2025-03-05T15:15:26.936946Z","iopub.status.idle":"2025-03-05T15:17:09.030881Z","shell.execute_reply.started":"2025-03-05T15:15:26.936918Z","shell.execute_reply":"2025-03-05T15:17:09.029965Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Great, so now the loss is going down and the model is training correctly.\n\nNow there are 2 more things we need to implement: Early Stopping, and tracking metrics like Accuracy, Recall, Precision and F1 score.\n\nAccuracy = What percentage of predictions your model can make correctly\nPrecision = What percentage of times when the model predicts a label is it correct\nRecall = What percentage of each label did the model correctly predict\nF1 - Harmonic Mean of Precision and Recall","metadata":{}},{"cell_type":"code","source":"# You cannot calculate metrics on your training data, and you can't do it on the testing set as well,\n# so we first need to create a training and validation set as well\ntrain_split = int(0.8 * len(train_dataset))\nval_split = int(0.2 * len(train_dataset))\ntrain_split, val_split\n\ntrain_dataset, val_dataset = random_split(train_dataset, [train_split, val_split])\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:48.010236Z","iopub.execute_input":"2025-03-05T15:37:48.010531Z","iopub.status.idle":"2025-03-05T15:37:48.022604Z","shell.execute_reply.started":"2025-03-05T15:37:48.010508Z","shell.execute_reply":"2025-03-05T15:37:48.021828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Early Stopping and Metrics Calculation","metadata":{}},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience = 5, min_change = 0.001, checkpoint_pth = \"best_model.pth\"):\n        self.patience = patience\n        self.min_change = min_change\n        self.best_loss = float(\"inf\")\n        self.counter = 0\n        self.early_stop = False\n        self.checkpoint_pth = checkpoint_pth\n\n    def __call__(self, val_loss, model):\n        if val_loss < self.best_loss - self.min_change:\n            counter = 0\n            self.best_loss = val_loss\n\n            torch.save(model.state_dict(), self.checkpoint_pth)\n        else:\n            self.counter += 1\n\n            print(f\"Early Stopping Counter: {self.counter}/{self.patience}\")\n            \n            if self.counter >= self.patience:\n                self.early_stop = True\n\n    def reset(self):\n        self.counter = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:53.770988Z","iopub.execute_input":"2025-03-05T15:37:53.771346Z","iopub.status.idle":"2025-03-05T15:37:53.777342Z","shell.execute_reply.started":"2025-03-05T15:37:53.771317Z","shell.execute_reply":"2025-03-05T15:37:53.776424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_metrics(model, val_loader, device):\n    model.eval()    # Set the model to evaluation mode\n\n    all_labels = []\n    all_preds = []\n    \n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.to(device), labels.to(device)\n\n            outputs = model(images)\n            preds = torch.argmax(F.softmax(outputs, dim=1), dim=1)\n\n            all_preds.extend(preds.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n\n    accuracy = accuracy_score(all_labels, all_preds)\n    precision = precision_score(all_labels, all_preds, average='weighted', zero_division=0)\n    recall = recall_score(all_labels, all_preds, average='weighted', zero_division=0)\n    f1 = f1_score(all_labels, all_preds, average='weighted', zero_division=0)\n\n    return accuracy, precision, recall, f1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:37:57.450802Z","iopub.execute_input":"2025-03-05T15:37:57.451084Z","iopub.status.idle":"2025-03-05T15:37:57.456634Z","shell.execute_reply.started":"2025-03-05T15:37:57.451062Z","shell.execute_reply":"2025-03-05T15:37:57.455907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating a more complex training function, with EarlyStopping and calculating all evaluation metrics\nearly_stopper = EarlyStopping()\n\ndef train(model, train_loader, val_loader, loss_fn, optimizer, early_stopper, device, n_epochs=5):\n    print(f\"Device being used: {device}\")\n    \n    model.to(device)\n    train_losses = []\n    val_losses = []\n    \n    for epoch_num in range(1, n_epochs+1):\n        start_time = timer()\n        model.train()    # Set the model to training model\n        train_loss = 0.0\n\n        # Training step\n        for images, labels in train_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n    \n            optimizer.zero_grad()            # Reset gradients to 0\n\n            outputs = model(images)           # Forward pass\n            loss = loss_fn(outputs, labels)\n            loss.backward()                 # Start Backprop\n            \n            optimizer.step()                 # Update gradients\n    \n            train_loss += loss.item()\n    \n        train_loss /= len(train_loader)\n        accuracy, precision, recall, f1 = calculate_metrics(model, val_loader, device)\n\n        train_losses.append(train_loss)\n    \n        # Validation step\n        model.eval()    # Set the model to evaluation mode\n        val_loss = 0.0\n        \n        with torch.no_grad():\n            for images, labels in val_loader:\n                images, labels = images.to(device), labels.to(device)\n                print(images.dim())\n                outputs = model(images)\n                loss = loss_fn(outputs, labels)\n                val_loss += loss.item()\n\n        val_loss /= len(val_loader)\n        \n        val_losses.append(val_loss)\n        \n        print(\n            f\"Epoch {epoch_num}/{n_epochs};  Time: {timer()-start_time:.4f}; Train Loss = {train_loss:.4f};\",\n            f\"Validation Loss: {val_loss:.4f};\",\n            f\"Accuracy: {accuracy:.4f}; Precision: {precision:.4f}; Recall: {recall:.4f}; F1: {f1:.4f}; \"\n        )\n        \n        # Call Early Stopping\n        early_stopper(val_loss, model)\n        \n        if early_stopper.early_stop:\n            print(f\"Early Stopping Triggered. Best model saved at {early_stopper.checkpoint_pth}\")\n            break\n\n    return train_losses, val_losses","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:44:00.313375Z","iopub.execute_input":"2025-03-05T15:44:00.313748Z","iopub.status.idle":"2025-03-05T15:44:00.321211Z","shell.execute_reply.started":"2025-03-05T15:44:00.313719Z","shell.execute_reply":"2025-03-05T15:44:00.320294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_losses, val_losses = train(model, train_loader, val_loader, loss, optimizer, early_stopper, device, 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:17:09.722112Z","iopub.execute_input":"2025-03-05T15:17:09.722412Z","iopub.status.idle":"2025-03-05T15:20:22.832014Z","shell.execute_reply.started":"2025-03-05T15:17:09.722382Z","shell.execute_reply":"2025-03-05T15:20:22.831008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_train_and_val_losses(train_losses, val_losses):\n    plt.plot(train_losses, color='b', label='Training Loss')\n    plt.plot(val_losses, color='g', label='Validation Loss')\n\n    plt.title(\"Loss VS Epochs\")\n    plt.xlabel(\"Number of Epochs\")\n    plt.ylabel(\"Loss Value\")\n    plt.legend()\n    \n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:38:17.807369Z","iopub.execute_input":"2025-03-05T15:38:17.807664Z","iopub.status.idle":"2025-03-05T15:38:17.811961Z","shell.execute_reply.started":"2025-03-05T15:38:17.807637Z","shell.execute_reply":"2025-03-05T15:38:17.811221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"So, this model is showing clear signs of overfitting. Let's apply weight decay, and increase the Dropout rate. The training function, EarlyStopping and calculate_metrics functions are working correctly, so let's adjust the basic architecture. ","metadata":{}},{"cell_type":"code","source":"class CNN2(nn.Module):\n    def __init__(self, dropout_prob=0.5):\n        super(CNN2, self).__init__()\n        self.conv_layers = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1),  # Conv Layer 1\n            nn.ReLU(),\n            nn.Conv2d(32, 64, kernel_size=3, padding=1),  # Conv Layer 2\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),  # Max Pooling 1\n            \n            nn.Conv2d(64, 128, kernel_size=3, padding=1),  # Conv Layer 3\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)  # Max Pooling 2\n        )\n        \n        self.fc_layers = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 8 * 8, 256),  # Fully connected layer 1\n            nn.ReLU(),\n            nn.Linear(256, 128),  # Fully connected layer 2\n            nn.ReLU(),\n            nn.Dropout(dropout_prob),  # Dropout for regularization, increased to 0.5\n            nn.Linear(128, 10)  # Output layer (10 classes)\n        )\n    \n    def forward(self, x):\n        x = self.conv_layers(x)\n        x = self.fc_layers(x)\n        return x\n\n# Initialize model\nmodel2 = CNN2()\nprint(model2)\n\n# Adding weight decay\noptimizer = optim.Adam(model2.parameters(), lr=0.001, weight_decay=0.0001)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:38:27.753644Z","iopub.execute_input":"2025-03-05T15:38:27.753965Z","iopub.status.idle":"2025-03-05T15:38:27.781385Z","shell.execute_reply.started":"2025-03-05T15:38:27.753938Z","shell.execute_reply":"2025-03-05T15:38:27.780683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's train this new model, and check how it fares\nearly_stopper2 = EarlyStopping(checkpoint_pth='best_model_v2.pth')\n\ntrain_losses, val_losses = train(model2, train_loader, val_loader, loss, optimizer, early_stopper2, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:41:44.808937Z","iopub.execute_input":"2025-03-05T15:41:44.809279Z","iopub.status.idle":"2025-03-05T15:41:52.355359Z","shell.execute_reply.started":"2025-03-05T15:41:44.809251Z","shell.execute_reply":"2025-03-05T15:41:52.354218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Model training absolutely poorly. Now it's underfitting, not learning at all. Likely a cause of high weight decay, and low learning rate.\n\nTrying a manual grid search of hyperparameters: lr, weight_decay, dropout_prob","metadata":{}},{"cell_type":"code","source":"# Dropout rate 0.3, LR = 1e-3, Weight Decay = 1e-10\nmodel3 = CNN2(0.3)\noptimizer3 = optim.Adam(model3.parameters(), lr=1e-3, weight_decay=1e-10)\n\nearly_stopper3 = EarlyStopping(checkpoint_pth='best_model_v3.pth')\n\ntrain_losses, val_losses = train(model3, train_loader, val_loader, loss, optimizer3, early_stopper3, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-03-05T15:26:38.501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dropout rate 0.3, LR = 1e-1, Weight Decay = 1e-4\nmodel4 = CNN2(0.3)\noptimizer4 = optim.Adam(model4.parameters(), lr=1e-1, weight_decay=1e-4)\n\nearly_stopper4 = EarlyStopping(checkpoint_pth='best_model_v4.pth')\n\ntrain_losses, val_losses = train(model4, train_loader, val_loader, loss, optimizer4, early_stopper4, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-03-05T15:26:38.501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dropout rate 0.5, LR = 1e-3, Weight Decay = 1e-10\nmodel5 = CNN2(0.5)\noptimizer5 = optim.Adam(model5.parameters(), lr=1e-3, weight_decay=1e-10)\n\nearly_stopper5 = EarlyStopping(checkpoint_pth='best_model_v5.pth')\n\ntrain_losses, val_losses = train(model5, train_loader, val_loader, loss, optimizer5, early_stopper5, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-03-05T15:26:38.501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dropout rate 0.5, LR = 1e-1, Weight Decay = 1e-4\nmodel6 = CNN2(0.5)\noptimizer6 = optim.Adam(model6.parameters(), lr=1e-1, weight_decay=1e-4)\n\nearly_stopper6 = EarlyStopping(checkpoint_pth='best_model_v5.pth')\n\ntrain_losses, val_losses = train(model6, train_loader, val_loader, loss, optimizer6, early_stopper6, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-03-05T15:26:38.502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dropout rate 0.3, LR = 1e-3, Weight Decay = 1e-4\nmodel7 = CNN2(0.3)\noptimizer7 = optim.Adam(model7.parameters(), lr=1e-3, weight_decay=1e-4)\n\nearly_stopper7 = EarlyStopping(checkpoint_pth='best_model_v5.pth')\n\ntrain_losses, val_losses = train(model7, train_loader, val_loader, loss, optimizer7, early_stopper7, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-03-05T15:26:38.502Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Removing weight decay, adding BatchNorm after every layer, and reducing Dropout","metadata":{}},{"cell_type":"code","source":"class CNN3(nn.Module):\n    def __init__(self, dropout_prob=0.3):\n        super(CNN3, self).__init__()\n        self.conv_layers = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1),  # Conv Layer 1\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.Conv2d(32, 64, kernel_size=3, padding=1),  # Conv Layer 2\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),  # Max Pooling 1\n            \n            nn.Conv2d(64, 128, kernel_size=3, padding=1),  # Conv Layer 3\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)  # Max Pooling 2\n        )\n        \n        self.fc_layers = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 8 * 8, 256),  # Fully connected layer 1\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Linear(256, 128),  # Fully connected layer 2\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(dropout_prob),  # Dropout for regularization, increased to 0.5\n            nn.Linear(128, 10)  # Output layer (10 classes)\n        )\n    \n    def forward(self, x):\n        x = self.conv_layers(x)\n        x = self.fc_layers(x)\n        \n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:52:03.596516Z","iopub.execute_input":"2025-03-05T15:52:03.596818Z","iopub.status.idle":"2025-03-05T15:52:03.603087Z","shell.execute_reply.started":"2025-03-05T15:52:03.596795Z","shell.execute_reply":"2025-03-05T15:52:03.602389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating a more complex training function, with EarlyStopping and calculating all evaluation metrics\ndef train(model, train_loader, val_loader, loss_fn, optimizer, early_stopper, device, n_epochs=5):\n    print(f\"Device being used: {device}\")\n    \n    model.to(device)\n    train_losses = []\n    val_losses = []\n    \n    for epoch_num in range(1, n_epochs+1):\n        start_time = timer()\n        model.train()    # Set the model to training model\n        train_loss = 0.0\n\n        # Training step\n        for images, labels in train_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n    \n            optimizer.zero_grad()             # Reset gradients to 0\n\n            outputs = model(images)           # Forward pass\n            loss = loss_fn(outputs, labels)\n            loss.backward()                   # Start Backprop\n            \n            optimizer.step()                  # Update gradients\n    \n            train_loss += loss.item()\n    \n        train_loss /= len(train_loader)\n        accuracy, precision, recall, f1 = calculate_metrics(model, val_loader, device)\n\n        train_losses.append(train_loss)\n    \n        # Validation step\n        model.eval()    # Set the model to evaluation mode\n        val_loss = 0.0\n        \n        with torch.no_grad():\n            for images, labels in val_loader:\n                images, labels = images.to(device), labels.to(device)\n\n                outputs = model(images)\n                loss = loss_fn(outputs, labels)\n                val_loss += loss.item()\n\n        val_loss /= len(val_loader)\n        \n        val_losses.append(val_loss)\n        \n        print(\n            f\"Epoch {epoch_num}/{n_epochs} |  Time: {timer()-start_time:.2f} | Train Loss = {train_loss:.4f} |\",\n            f\"Validation Loss: {val_loss:.4f} |\",\n            f\"Accuracy: {accuracy:.4f} | Precision: {precision:.4f} | Recall: {recall:.4f} | F1: {f1:.4f}; \"\n        )\n        \n        # Call Early Stopping\n        early_stopper(val_loss, model)\n        \n        if early_stopper.early_stop:\n            print(f\"Early Stopping Triggered. Best model saved at {early_stopper.checkpoint_pth}\")\n            break\n\n    return train_losses, val_losses","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T16:03:50.024958Z","iopub.execute_input":"2025-03-05T16:03:50.025173Z","iopub.status.idle":"2025-03-05T16:03:50.032479Z","shell.execute_reply.started":"2025-03-05T16:03:50.025154Z","shell.execute_reply":"2025-03-05T16:03:50.031578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Reducing the Dropout probability\nmodel3_1 = CNN3(dropout_prob=0.2)\noptimizer3_1 = optim.Adam(model3_1.parameters(), lr=5e-4)\n\nearlyStopper3_1 = EarlyStopping(checkpoint_pth = 'best_model3_1.pth')\n\ntrain(model3_1, train_loader, val_loader, loss, optimizer3_1, earlyStopper3_1, device, 10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:52:03.970349Z","iopub.execute_input":"2025-03-05T15:52:03.970604Z","iopub.status.idle":"2025-03-05T15:57:15.367637Z","shell.execute_reply.started":"2025-03-05T15:52:03.970585Z","shell.execute_reply":"2025-03-05T15:57:15.366761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Increasing the Dropout probability alongside BatchNorm\nmodel3_2 = CNN3(dropout_prob=0.5)\noptimizer3_2 = optim.Adam(model3_2.parameters(), lr=5e-4)\n\nearlyStopper3_2 = EarlyStopping(checkpoint_pth = 'best_model3_2.pth')\n\ntrain(model3_2, train_loader, val_loader, loss, optimizer3_2, earlyStopper3_2, device, 10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T15:57:15.368746Z","iopub.execute_input":"2025-03-05T15:57:15.368983Z","iopub.status.idle":"2025-03-05T16:03:50.023557Z","shell.execute_reply.started":"2025-03-05T15:57:15.368962Z","shell.execute_reply":"2025-03-05T16:03:50.022386Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Trend shows that increasing the Dropout rate alongside having BatchNorm is making the training smoother. Let's test the theory","metadata":{}},{"cell_type":"code","source":"model3_3 = CNN3(dropout_prob=0.7)\noptimizer3_3 = optim.Adam(model3_3.parameters(), lr=5e-4)\n\nearlyStopper3_3 = EarlyStopping(checkpoint_pth = 'best_model3_3.pth')\n\ntrain_losses, val_losses = train(model3_3, train_loader, val_loader, loss, optimizer3_3, earlyStopper3_3, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T16:11:44.216258Z","iopub.execute_input":"2025-03-05T16:11:44.216608Z","iopub.status.idle":"2025-03-05T16:18:17.261753Z","shell.execute_reply.started":"2025-03-05T16:11:44.216584Z","shell.execute_reply":"2025-03-05T16:18:17.260872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Increasing the Learning Rate\nmodel3_3 = CNN3(dropout_prob=0.7)\noptimizer3_3 = optim.Adam(model3_3.parameters(), lr=1e-3)\n\nearlyStopper3_3 = EarlyStopping(checkpoint_pth = 'best_model3_3.pth')\n\ntrain_losses, val_losses = train(model3_3, train_loader, val_loader, loss, optimizer3_3, earlyStopper3_3, device, 10)\nplot_train_and_val_losses(train_losses, val_losses)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T16:19:56.269344Z","iopub.execute_input":"2025-03-05T16:19:56.269764Z","iopub.status.idle":"2025-03-05T16:26:32.356609Z","shell.execute_reply.started":"2025-03-05T16:19:56.269735Z","shell.execute_reply":"2025-03-05T16:26:32.355784Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"So accuracy is increasing on the validation set, but the disparity between the losses is increasing.\n\nSo far the best model is the last one that was trained, stored in \"best_model3_3.pth\"","metadata":{}},{"cell_type":"markdown","source":"### Evaluating on the test set","metadata":{}},{"cell_type":"code","source":"# Unpacking the test images, and applying transforms\n!7z x \"/kaggle/input/cifar-10/test.7z\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T16:30:47.987248Z","iopub.execute_input":"2025-03-05T16:30:47.987537Z","iopub.status.idle":"2025-03-05T16:47:36.816693Z","shell.execute_reply.started":"2025-03-05T16:30:47.987516Z","shell.execute_reply":"2025-03-05T16:47:36.815775Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating a custom Dataset class and the DataLoader\n\nclass CustomTestDataset(Dataset):\n    def __init__(self, images_dir, transform=None):\n        self.images_dir = images_dir\n        self.transform = transform\n        self.img_files = sorted(os.listdir(images_dir))  # Ensure correct order\n\n    def __len__(self):\n        return len(self.img_files)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.images_dir, self.img_files[idx])\n        image = Image.open(img_path).convert(\"RGB\")  # Open image as RGB\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, self.img_files[idx]\n\n\ntest_dataset = CustomTestDataset(\"/kaggle/working/test\", test_transform)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T17:06:02.286723Z","iopub.execute_input":"2025-03-05T17:06:02.287023Z","iopub.status.idle":"2025-03-05T17:06:02.351170Z","shell.execute_reply.started":"2025-03-05T17:06:02.287001Z","shell.execute_reply":"2025-03-05T17:06:02.350558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Loading best model\nmodel = CNN3()\nmodel.load_state_dict(torch.load(\"best_model3_3.pth\"))  \nmodel.to(device)\nmodel.eval()     # Set to evaluation mode","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T17:06:03.522080Z","iopub.execute_input":"2025-03-05T17:06:03.522391Z","iopub.status.idle":"2025-03-05T17:06:03.573625Z","shell.execute_reply.started":"2025-03-05T17:06:03.522367Z","shell.execute_reply":"2025-03-05T17:06:03.572887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions\nall_preds = []\nimg_idxs = []\n\nwith torch.no_grad():\n    for images, idxs in test_loader:\n        images = images.to(device)\n\n        outputs = model(images)\n\n        _, preds = torch.max(outputs, 1)\n\n        all_preds.extend(preds.cpu().numpy())\n        img_idxs.extend(idxs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T17:06:05.675621Z","iopub.execute_input":"2025-03-05T17:06:05.675940Z","iopub.status.idle":"2025-03-05T17:06:43.425150Z","shell.execute_reply.started":"2025-03-05T17:06:05.675913Z","shell.execute_reply":"2025-03-05T17:06:43.423923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Re-mapping predictions\nidxs_to_classes = {v: k for k, v in label_mapping.items()}\n\npredicted_labels = [idxs_to_classes[pred] for pred in all_preds]\n\nsubmission_df = pd.DataFrame({\n    \"id\": [int(file.split(\".\")[0]) for file in img_idxs],  \n    \"label\": predicted_labels\n})\n\n# Sort by ID to ensure correct order\nsubmission_df = submission_df.sort_values(by=\"id\")\n\n# Save to CSV\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T17:06:44.502591Z","iopub.execute_input":"2025-03-05T17:06:44.502910Z","iopub.status.idle":"2025-03-05T17:06:44.627802Z","shell.execute_reply.started":"2025-03-05T17:06:44.502883Z","shell.execute_reply":"2025-03-05T17:06:44.626871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T17:06:45.834869Z","iopub.execute_input":"2025-03-05T17:06:45.835161Z","iopub.status.idle":"2025-03-05T17:06:45.843626Z","shell.execute_reply.started":"2025-03-05T17:06:45.835139Z","shell.execute_reply":"2025-03-05T17:06:45.842832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}