{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":396.293132,"end_time":"2024-08-05T17:53:14.019222","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-05T17:46:37.726090","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle\nfrom PIL import Image\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport os\nimport torch.nn.functional as F\nimport requests\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport seaborn as sn\ntorch.manual_seed(1) #Set manual seed\nfrom sklearn import preprocessing\nfrom torch.utils.data import DataLoader, Dataset, Subset\nimport torch.optim as optim\nfrom sklearn.metrics import roc_auc_score, average_precision_score, balanced_accuracy_score, f1_score, accuracy_score, recall_score, precision_score, roc_curve, auc\nfrom time import time\nfrom sklearn.preprocessing import label_binarize\nimport torchvision.models as models\n# from torchvision.models import resnet10\\\nfrom sklearn.decomposition import PCA\n\n#Define GPU device\nmps_device = torch.device(\"mps\")\n\nimport torch\nimport math\nimport torch.nn as nn\nfrom torch.nn.parameter import Parameter\nimport time\n\nimport pdb\n\nimport math\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from layers.ModulatedAttLayer import ModulatedAttLayer","metadata":{"papermill":{"duration":7.545298,"end_time":"2024-08-05T17:46:48.072274","exception":false,"start_time":"2024-08-05T17:46:40.526976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:20.771929Z","iopub.execute_input":"2024-08-05T20:25:20.772289Z","iopub.status.idle":"2024-08-05T20:25:26.681045Z","shell.execute_reply.started":"2024-08-05T20:25:20.772259Z","shell.execute_reply":"2024-08-05T20:25:26.680208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation","metadata":{"papermill":{"duration":0.007863,"end_time":"2024-08-05T17:46:48.088316","exception":false,"start_time":"2024-08-05T17:46:48.080453","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IMAGE_SIZE = 32\nmean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261]\n# These values are mostly used by researchers as found to very useful in fast convergence\n\n# We can try various transformation for good generalization of model.\ncomposed_train = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), # Resize the image in a 32 x 32 shape\n                                     transforms.RandomRotation(20), # Randomly rotate some images by 20 degrees\n                                     transforms.RandomHorizontalFlip(0.1), #Randomly horizontal flip the images\n                                     transforms.ColorJitter(brightness = 0.1, #Randomly adjust color jitter of the images\n                                                            contrast = 0.1,\n                                                            saturation = 0.1),\n                                     transforms.RandomAdjustSharpness(sharpness_factor = 2,\n                                                                      p = 0.1), # Randomly adjust sharpness\n                                     transforms.ToTensor(), # Converting image to tensor\n                                     transforms.Normalize(mean, std), # Normalizing with standard mean and standard deviation\n                                     transforms.RandomErasing(p = 0.75, scale = (0.02, 0.1), value = 1.0, inplace = False)\n                                    ])\n\ncomposed_test = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize(mean, std)\n                                   ])","metadata":{"papermill":{"duration":0.020092,"end_time":"2024-08-05T17:46:48.115930","exception":false,"start_time":"2024-08-05T17:46:48.095838","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:26.682812Z","iopub.execute_input":"2024-08-05T20:25:26.683229Z","iopub.status.idle":"2024-08-05T20:25:26.691909Z","shell.execute_reply.started":"2024-08-05T20:25:26.683203Z","shell.execute_reply":"2024-08-05T20:25:26.690959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCIFAR10(datasets.CIFAR10):\n    def __getitem__(self, index):\n        img, target = super().__getitem__(index)\n        return img, target, index\n    \n#Custom Collate Function\ndef custom_collate_fn(batch):\n    data = torch.stack([item[0] for item in batch])\n    targets = torch.tensor([item[1] for item in batch])\n    indices = torch.tensor([item[2] for item in batch])\n    return data, targets, indices","metadata":{"papermill":{"duration":0.01719,"end_time":"2024-08-05T17:46:48.140755","exception":false,"start_time":"2024-08-05T17:46:48.123565","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:26.693093Z","iopub.execute_input":"2024-08-05T20:25:26.693418Z","iopub.status.idle":"2024-08-05T20:25:26.704145Z","shell.execute_reply.started":"2024-08-05T20:25:26.693388Z","shell.execute_reply":"2024-08-05T20:25:26.703215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data and transform the dataset\ntrain_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_train)\nvalidation_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_test)\n\n# Create train and validation batch for training\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=100, shuffle=True, num_workers=2, collate_fn=custom_collate_fn)\nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=200, shuffle=False, num_workers=2, collate_fn=custom_collate_fn)","metadata":{"papermill":{"duration":15.093235,"end_time":"2024-08-05T17:47:03.241487","exception":false,"start_time":"2024-08-05T17:46:48.148252","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:26.706698Z","iopub.execute_input":"2024-08-05T20:25:26.707046Z","iopub.status.idle":"2024-08-05T20:25:36.355384Z","shell.execute_reply.started":"2024-08-05T20:25:26.707016Z","shell.execute_reply":"2024-08-05T20:25:36.354443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_data(img):\n    img = im_convert(img)\n    plt.imshow(img)\n    plt.show()\n    \n# We need to convert the images to numpy arrays as tensors are not compatible with matplotlib\ndef im_convert(tensor):\n    img = tensor.cpu().clone().detach().numpy()\n    img = img.transpose(1,2,0)\n    img = img*np.array(tuple(mean)) + np.array(tuple(std))\n    img = img.clip(0,1) #Clipping the size to print the images later\n    return img","metadata":{"papermill":{"duration":0.020024,"end_time":"2024-08-05T17:47:03.273561","exception":false,"start_time":"2024-08-05T17:47:03.253537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:36.356519Z","iopub.execute_input":"2024-08-05T20:25:36.356815Z","iopub.status.idle":"2024-08-05T20:25:36.363082Z","shell.execute_reply.started":"2024-08-05T20:25:36.356791Z","shell.execute_reply":"2024-08-05T20:25:36.361912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Visualization","metadata":{"papermill":{"duration":0.011117,"end_time":"2024-08-05T17:47:03.296094","exception":false,"start_time":"2024-08-05T17:47:03.284977","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # Different classes in CIFAR10 dataset:\n# classes = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n# # Define an iterable on the data\n# data_iterable = iter(train_loader) # Converting our train_dataloader to iterable so that we can iterate through it.\n# images, labels = next(data_iterable) # Going from 1st batch of 100 images to the next batch\n# fig = plt.figure(figsize=(20,10))\n\n# # Let's plot 50 images from our train_dataset\n# for idx in np.arange(10):\n#     ax = fig.add_subplot(2, 5, idx+1, xticks=[], yticks=[])\n    \n#     # Note: imshow can't print tensor\n#     # Let's convert tensor image to numpy using im_convert function for imshow to print the image\n#     plt.imshow(im_convert(images[idx]))\n#     ax.set_title(classes[labels[idx].item()])","metadata":{"papermill":{"duration":0.018284,"end_time":"2024-08-05T17:47:03.326604","exception":false,"start_time":"2024-08-05T17:47:03.308320","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:36.364093Z","iopub.execute_input":"2024-08-05T20:25:36.364347Z","iopub.status.idle":"2024-08-05T20:25:36.373930Z","shell.execute_reply.started":"2024-08-05T20:25:36.364324Z","shell.execute_reply":"2024-08-05T20:25:36.373074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Classifier**","metadata":{"papermill":{"duration":0.011195,"end_time":"2024-08-05T17:47:03.349195","exception":false,"start_time":"2024-08-05T17:47:03.338000","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CosNorm_Classifier(nn.Module):\n    def __init__(self, in_dims, out_dims, scale=10.0):\n        super(CosNorm_Classifier, self).__init__()\n        self.scale = scale\n        self.weight = nn.Parameter(torch.Tensor(out_dims, in_dims))\n        nn.init.xavier_uniform_(self.weight) \n\n#     def reset_parameters(self):\n#         stdv = 1. / math.sqrt(self.weight.size(1))\n#         self.weight.data.uniform_(-stdv, stdv)\n\n    def forward(self, input):\n#         print(f\"Input shape to CosNorm_Classifier: {input.shape}\")  # Debug print\n        norm_x = torch.norm(input.clone(), 2, 1, keepdim=True)\n        ex = (norm_x / (1 + norm_x)) * (input / norm_x)\n        ew = self.weight / torch.norm(self.weight, 2, 1, keepdim=True)\n#         print(f\"Shape of ex: {ex.shape}, Shape of ew: {ew.shape}\")  # Debug print\n        return torch.mm(self.scale * ex, ew.t())","metadata":{"papermill":{"duration":0.021397,"end_time":"2024-08-05T17:47:03.382551","exception":false,"start_time":"2024-08-05T17:47:03.361154","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:36.375227Z","iopub.execute_input":"2024-08-05T20:25:36.375971Z","iopub.status.idle":"2024-08-05T20:25:36.386367Z","shell.execute_reply.started":"2024-08-05T20:25:36.375939Z","shell.execute_reply":"2024-08-05T20:25:36.385587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Necessary functions for Triplet Loss","metadata":{"papermill":{"duration":0.011321,"end_time":"2024-08-05T17:47:03.405490","exception":false,"start_time":"2024-08-05T17:47:03.394169","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# Define the SupConLoss\nclass SupConLoss(nn.Module):\n    def __init__(self, temperature=0.07, contrast_mode='all', base_temperature=0.07):\n        super(SupConLoss, self).__init__()\n        self.temperature = temperature\n        self.contrast_mode = contrast_mode\n        self.base_temperature = base_temperature\n\n    def forward(self, features, labels=None, mask=None):\n        device = (torch.device('cuda') if features.is_cuda else torch.device('cpu'))\n        \n        if len(features.shape) < 3:\n            raise ValueError('`features` needs to be [bsz, n_views, ...], at least 3 dimensions are required')\n        if len(features.shape) > 3:\n            features = features.view(features.shape[0], features.shape[1], -1)\n        \n        batch_size = features.shape[0]\n        if labels is not None and mask is not None:\n            raise ValueError('Cannot define both `labels` and `mask`')\n        elif labels is None and mask is None:\n            mask = torch.eye(batch_size, dtype=torch.float32).to(device)\n        elif labels is not None:\n            labels = labels.contiguous().view(-1, 1)\n            if labels.shape[0] != batch_size:\n                raise ValueError('Num of labels does not match num of features')\n            mask = torch.eq(labels, labels.T).float().to(device)\n        else:\n            mask = mask.float().to(device)\n\n        contrast_count = features.shape[1]\n        contrast_feature = torch.cat(torch.unbind(features, dim=1), dim=0)\n        if self.contrast_mode == 'one':\n            anchor_feature = features[:, 0]\n            anchor_count = 1\n        elif self.contrast_mode == 'all':\n            anchor_feature = contrast_feature\n            anchor_count = contrast_count\n        else:\n            raise ValueError('Unknown mode: {}'.format(self.contrast_mode))\n\n        # Compute logits\n        anchor_dot_contrast = torch.div(\n            torch.matmul(anchor_feature, contrast_feature.T),\n            self.temperature)\n        # For numerical stability\n        logits_max, _ = torch.max(anchor_dot_contrast, dim=1, keepdim=True)\n        logits = anchor_dot_contrast - logits_max.detach()\n\n        # Tile mask\n        mask = mask.repeat(anchor_count, contrast_count)\n        # Mask-out self-contrast cases\n        logits_mask = torch.scatter(\n            torch.ones_like(mask),\n            1,\n            torch.arange(batch_size * anchor_count).view(-1, 1).to(device),\n            0\n        )\n        mask = mask * logits_mask\n\n        # Compute log_prob\n        exp_logits = torch.exp(logits) * logits_mask\n        log_prob = logits - torch.log(exp_logits.sum(1, keepdim=True))\n\n        # Compute mean of log-likelihood over positive\n        mask_pos_pairs = mask.sum(1)\n        mask_pos_pairs = torch.where(mask_pos_pairs < 1e-6, 1, mask_pos_pairs)\n        mean_log_prob_pos = (mask * log_prob).sum(1) / mask_pos_pairs\n\n        # Loss\n        loss = - (self.temperature / self.base_temperature) * mean_log_prob_pos\n        loss = loss.view(anchor_count, batch_size).mean()\n\n        return loss\n","metadata":{"papermill":{"duration":0.026633,"end_time":"2024-08-05T17:47:03.445068","exception":false,"start_time":"2024-08-05T17:47:03.418435","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:25:36.387389Z","iopub.execute_input":"2024-08-05T20:25:36.387643Z","iopub.status.idle":"2024-08-05T20:25:36.405143Z","shell.execute_reply.started":"2024-08-05T20:25:36.387622Z","shell.execute_reply":"2024-08-05T20:25:36.404311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Base network creation function\nclass BaseNetwork(nn.Module):\n    def __init__(self, embedding_size):\n        super(BaseNetwork, self).__init__()\n        self.embedding_size = embedding_size\n        self.flatten = nn.Flatten()\n        self.fc1 = nn.Linear(3*32*32, 128)\n        self.dropout1 = nn.Dropout(0.1)\n        self.fc2 = nn.Linear(128, 128)\n        self.dropout2 = nn.Dropout(0.1)\n        self.fc3 = nn.Linear(128, embedding_size)\n        \n    def forward(self, x):\n        x = self.flatten(x)\n        x = F.relu(self.fc1(x))\n        x = self.dropout1(x)\n        x = F.relu(self.fc2(x))\n        x = self.dropout2(x)\n        x = self.fc3(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-08-05T20:30:25.394136Z","iopub.execute_input":"2024-08-05T20:30:25.394483Z","iopub.status.idle":"2024-08-05T20:30:25.402453Z","shell.execute_reply.started":"2024-08-05T20:30:25.394456Z","shell.execute_reply":"2024-08-05T20:30:25.401443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ExtendedNetwork(nn.Module):\n    def __init__(self, base_network, embedding_dim, num_classes):\n        super(ExtendedNetwork, self).__init__()\n        self.base_network = base_network\n        self.classifier = nn.Linear(embedding_dim, num_classes)\n\n    def forward(self, x):\n        embeddings = self.base_network(x)\n        scores = self.classifier(embeddings)\n        return embeddings, scores\n\n# Assuming the base network's embedding dimension is known (e.g., 128)\nembedding_dim = 128\nnum_classes = 10\n\nbase_network = BaseNetwork(embedding_size=embedding_dim)  # Your base network definition\nextended_network = ExtendedNetwork(base_network, embedding_dim, num_classes)","metadata":{"execution":{"iopub.status.busy":"2024-08-05T20:31:20.177599Z","iopub.execute_input":"2024-08-05T20:31:20.177983Z","iopub.status.idle":"2024-08-05T20:31:20.208886Z","shell.execute_reply.started":"2024-08-05T20:31:20.177953Z","shell.execute_reply":"2024-08-05T20:31:20.208070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model**","metadata":{"papermill":{"duration":0.013249,"end_time":"2024-08-05T17:47:03.471654","exception":false,"start_time":"2024-08-05T17:47:03.458405","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class SimpleCNN(nn.Module):\n    def __init__(self):\n        super(SimpleCNN, self).__init__()\n        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n        self.fc1 = nn.Linear(64*8*8, 128)\n        self.fc2 = nn.Linear(128, 10) #CIFAR-10 has 10 classes\n        \n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = F.max_pool2d(x, 2)\n        x = F.relu(self.conv2(x))\n        x = F.max_pool2d(x, 2)\n        x = x.view(x.size(0), -1)\n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)\n        \n        return x","metadata":{"papermill":{"duration":0.024715,"end_time":"2024-08-05T17:47:03.509655","exception":false,"start_time":"2024-08-05T17:47:03.484940","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:31:22.409147Z","iopub.execute_input":"2024-08-05T20:31:22.409504Z","iopub.status.idle":"2024-08-05T20:31:22.417175Z","shell.execute_reply.started":"2024-08-05T20:31:22.409478Z","shell.execute_reply":"2024-08-05T20:31:22.416266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialize model, Optimizer\n# embedded_size = 128\n# num_classes = 10\n# # model = SimpleCNN()\n# extended_network = ExtendedNetwork(BaseNetwork, embedding_dim, num_classes)\n# Initialize the SupCon loss function\nsupcon_loss_fn = SupConLoss()\noptimizer = optim.Adam(extended_network.parameters(), lr=0.0001)","metadata":{"papermill":{"duration":0.042008,"end_time":"2024-08-05T17:47:03.601096","exception":false,"start_time":"2024-08-05T17:47:03.559088","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:31:31.628670Z","iopub.execute_input":"2024-08-05T20:31:31.629534Z","iopub.status.idle":"2024-08-05T20:31:31.634306Z","shell.execute_reply.started":"2024-08-05T20:31:31.629504Z","shell.execute_reply":"2024-08-05T20:31:31.633295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get embeddings before training\nextended_network.eval()\nembeddings_before_train = []\nlabels_before_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        embeddings, _ = extended_network(data)\n#         embeddings = embeddings.unsqueeze(1) #Add the view dimension\n        embeddings_before_train.append(embeddings.cpu().numpy())\n        labels_before_train.append(target.cpu().numpy())\n        \nembeddings_before_train = np.concatenate(embeddings_before_train)\nlabels_before_train = np.concatenate(labels_before_train)","metadata":{"papermill":{"duration":7.954256,"end_time":"2024-08-05T17:47:11.566732","exception":false,"start_time":"2024-08-05T17:47:03.612476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T20:33:24.853368Z","iopub.execute_input":"2024-08-05T20:33:24.853799Z","iopub.status.idle":"2024-08-05T20:33:33.294036Z","shell.execute_reply.started":"2024-08-05T20:33:24.853764Z","shell.execute_reply":"2024-08-05T20:33:33.292835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Loop\nnum_epochs = 10\ntraining_loss = []\nextended_network.train()\nfor epoch in range(num_epochs):\n    base_network.train()\n    epoch_loss = 0\n    for data, target, index in train_loader:\n        optimizer.zero_grad()\n        embeddings, scores = extended_network(data)\n        \n        # Ensure the shape of embeddings is [batch_size, num_views, embedding_dim]\n        if embeddings.dim() == 2:\n            embeddings = embeddings.unsqueeze(1)  # Add the view dimension\n            \n        loss = supcon_loss_fn(embeddings, target)\n        loss.backward()\n        optimizer.step()\n        epoch_loss += loss.item()\n    training_loss.append(epoch_loss / len(train_loader))\n        \n    print(f'Epoch {epoch + 1}/{num_epochs}, Loss: {epoch_loss / len(train_loader): .4f}')\n    \n    # Evaluate in validation set\n    extended_network.eval()\n    all_targets = []\n    all_embeddings = []\n    all_predictions = []\n    all_scores = []\n    start_time = time.time()\n    with torch.no_grad():\n        for data, target, index in validation_loader:\n            embeddings, scores = extended_network(data)\n#             outputs = extended_network(data)\n#             embeddings = embeddings.unsqueeze(1)  # Add the view dimension\n            _, predicted = torch.max(scores, 1)\n            all_targets.extend(target.cpu().numpy())\n            all_predictions.extend(predicted.cpu().numpy())\n            all_embeddings.extend(embeddings.cpu().numpy())\n            all_scores.extend(scores.cpu().numpy())\n    end_time = time.time()\n    testing_time = end_time - start_time\n    \n#     # Validate the length of all_targets and all_predictions\n#     print(f'Length of all_targets: {len(all_targets)}')\n#     print(f'Length of all_predictions: {len(all_predictions)}')\n\n#     # Print the first few elements to debug\n#     print('First few elements of all_targets:', all_targets[:5])\n#     print('First few elements of all_predictions:', all_predictions[:5])\n\n#     # Make sure to use the correct predictions\n#     all_predictions = np.array(all_predictions)\n#     if len(all_targets) != len(all_predictions):\n#         print('Mismatch in lengths, adjusting prediction values')\n\n    \n    # Calculate metrics\n    accuracy = accuracy_score(all_targets, all_predictions)\n    balanced_acc = balanced_accuracy_score(all_targets, all_predictions)\n    f1 = f1_score(all_targets, all_predictions, average='weighted')\n    recall = recall_score(all_targets, all_predictions, average='weighted')\n    precision = precision_score(all_targets, all_predictions, average='weighted')\n    \n    print(f'Epoch {epoch+1}, Accuracy: {accuracy:.4f}, Balanced Accuracy: {balanced_acc:.4f},'\n          f'F1 Score: {f1: .4f}, Recall: {recall: .4f}, Precision: {precision: .4f},'\n          f'Testing Time: {testing_time: .4f}s')","metadata":{"papermill":{"duration":349.005713,"end_time":"2024-08-05T17:53:00.584683","exception":false,"start_time":"2024-08-05T17:47:11.578970","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T21:00:58.718527Z","iopub.execute_input":"2024-08-05T21:00:58.718971Z","iopub.status.idle":"2024-08-05T21:06:57.273194Z","shell.execute_reply.started":"2024-08-05T21:00:58.718933Z","shell.execute_reply":"2024-08-05T21:06:57.272083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After training, plot ROC Curve\n\n# Plot ROC Curve\nextended_network.eval()\nwith torch.no_grad():\n    all_targets = np.array(all_targets)\n    all_scores = np.array(all_scores)\n\n    # Binarize the labels for multi-class ROC computation\n    y_bin = label_binarize(all_targets, classes=np.arange(num_classes))\n\n    # Plot ROC curves for each class\n    plt.figure(figsize=(10, 8))\n    for i in range(num_classes):\n        fpr, tpr, _ = roc_curve(y_bin[:, i], all_scores[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label=f'Class {i} (area = {roc_auc:.2f})')\n\n    plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic for Multi-Class')\n    plt.legend(loc='lower right')\n    plt.show()","metadata":{"papermill":{"duration":0.644941,"end_time":"2024-08-05T17:53:01.243236","exception":false,"start_time":"2024-08-05T17:53:00.598295","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T21:08:58.593308Z","iopub.execute_input":"2024-08-05T21:08:58.594324Z","iopub.status.idle":"2024-08-05T21:08:59.195903Z","shell.execute_reply.started":"2024-08-05T21:08:58.594280Z","shell.execute_reply":"2024-08-05T21:08:59.194997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get embeddings after train\nextended_network.eval()\nembeddings_after_train = []\nlabels_after_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        embeddings, _ = extended_network(data)\n        embeddings_after_train.append(embeddings.cpu().numpy())\n        labels_after_train.append(target.cpu().numpy())\n\nembeddings_after_train = np.concatenate(embeddings_after_train)\nlabels_after_train = np.concatenate(labels_after_train)","metadata":{"papermill":{"duration":8.286997,"end_time":"2024-08-05T17:53:09.546376","exception":false,"start_time":"2024-08-05T17:53:01.259379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T21:10:15.596184Z","iopub.execute_input":"2024-08-05T21:10:15.596606Z","iopub.status.idle":"2024-08-05T21:10:23.749691Z","shell.execute_reply.started":"2024-08-05T21:10:15.596572Z","shell.execute_reply":"2024-08-05T21:10:23.748597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the effect of embeddings using PCA\nnum_components=2\nstep = 5\n\npca = PCA(n_components=num_components)\ndecomposed_before_train = pca.fit_transform(embeddings_before_train)\ndecomposed_after_train = pca.fit_transform(embeddings_after_train)\ntest_class_labels = np.unique(labels_after_train)\n\nfig = plt.figure(figsize=(16, 8))\nfor label in test_class_labels:\n    decomposed_before_class = decomposed_before_train[labels_before_train == label]\n    decomposed_after_class = decomposed_after_train[labels_after_train == label]\n    \n    plt.subplot(1, 2, 1)\n    plt.scatter(decomposed_before_class[:, 0], decomposed_before_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('Before Training (embeddings)')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.scatter(decomposed_after_class[:, 0], decomposed_after_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('After {num_epochs} Epochs (embeddings)')\n    plt.legend()\n    \nplt.show()","metadata":{"papermill":{"duration":2.438617,"end_time":"2024-08-05T17:53:12.001335","exception":false,"start_time":"2024-08-05T17:53:09.562718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T21:10:27.043331Z","iopub.execute_input":"2024-08-05T21:10:27.043776Z","iopub.status.idle":"2024-08-05T21:10:29.475585Z","shell.execute_reply.started":"2024-08-05T21:10:27.043713Z","shell.execute_reply":"2024-08-05T21:10:29.474633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the trained model\ntorch.save(base_network.state_dict(), 'cifar10_base_network.pth')","metadata":{"papermill":{"duration":0.03941,"end_time":"2024-08-05T17:53:12.066760","exception":false,"start_time":"2024-08-05T17:53:12.027350","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T21:10:37.056848Z","iopub.execute_input":"2024-08-05T21:10:37.057473Z","iopub.status.idle":"2024-08-05T21:10:37.066624Z","shell.execute_reply.started":"2024-08-05T21:10:37.057442Z","shell.execute_reply":"2024-08-05T21:10:37.065636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the training data\nplt.figure(figsize=(8,8))\nplt.plot(training_loss, label='Training Loss')\nplt.legend()\nplt.title('Training Loss')\nplt.show()","metadata":{"papermill":{"duration":0.323289,"end_time":"2024-08-05T17:53:12.416229","exception":false,"start_time":"2024-08-05T17:53:12.092940","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T21:10:38.025266Z","iopub.execute_input":"2024-08-05T21:10:38.025648Z","iopub.status.idle":"2024-08-05T21:10:38.265524Z","shell.execute_reply.started":"2024-08-05T21:10:38.025618Z","shell.execute_reply":"2024-08-05T21:10:38.264567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.026372,"end_time":"2024-08-05T17:53:12.469114","exception":false,"start_time":"2024-08-05T17:53:12.442742","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}