{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle\nfrom PIL import Image\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport os\nimport torch.nn.functional as F\nimport requests\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport seaborn as sn\ntorch.manual_seed(1) #Set manual seed\nfrom sklearn import preprocessing\nfrom torch.utils.data import DataLoader, Dataset, Subset\nimport torch.optim as optim\nfrom sklearn.metrics import roc_auc_score, average_precision_score, balanced_accuracy_score, f1_score, accuracy_score, recall_score, precision_score, roc_curve, auc\nfrom time import time\nfrom sklearn.preprocessing import label_binarize\nimport torchvision.models as models\n# from torchvision.models import resnet10\\\nfrom sklearn.decomposition import PCA\n\n#Define GPU device\nmps_device = torch.device(\"mps\")\n\nimport torch\nimport math\nimport torch.nn as nn\nfrom torch.nn.parameter import Parameter\nimport time\n\nimport pdb\n\nimport math\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from layers.ModulatedAttLayer import ModulatedAttLayer","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:44.442514Z","iopub.execute_input":"2024-08-17T09:45:44.443550Z","iopub.status.idle":"2024-08-17T09:45:50.420700Z","shell.execute_reply.started":"2024-08-17T09:45:44.443504Z","shell.execute_reply":"2024-08-17T09:45:50.419669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:50.422758Z","iopub.execute_input":"2024-08-17T09:45:50.423650Z","iopub.status.idle":"2024-08-17T09:45:50.458512Z","shell.execute_reply.started":"2024-08-17T09:45:50.423594Z","shell.execute_reply":"2024-08-17T09:45:50.457282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = 32\nmean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261]\n# These values are mostly used by researchers as found to very useful in fast convergence\n\n# We can try various transformation for good generalization of model.\ncomposed_train = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), # Resize the image in a 32 x 32 shape\n                                     transforms.RandomRotation(20), # Randomly rotate some images by 20 degrees\n                                     transforms.RandomHorizontalFlip(0.1), #Randomly horizontal flip the images\n                                     transforms.ColorJitter(brightness = 0.1, #Randomly adjust color jitter of the images\n                                                            contrast = 0.1,\n                                                            saturation = 0.1),\n                                     transforms.RandomAdjustSharpness(sharpness_factor = 2,\n                                                                      p = 0.1), # Randomly adjust sharpness\n                                     transforms.ToTensor(), # Converting image to tensor\n                                     transforms.Normalize(mean, std), # Normalizing with standard mean and standard deviation\n                                     transforms.RandomErasing(p = 0.75, scale = (0.02, 0.1), value = 1.0, inplace = False)\n                                    ])\n\ncomposed_test = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize(mean, std)\n                                   ])","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:50.460018Z","iopub.execute_input":"2024-08-17T09:45:50.460433Z","iopub.status.idle":"2024-08-17T09:45:50.472072Z","shell.execute_reply.started":"2024-08-17T09:45:50.460401Z","shell.execute_reply":"2024-08-17T09:45:50.471273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCIFAR10(datasets.CIFAR10):\n    def __getitem__(self, index):\n        img, target = super().__getitem__(index)\n        return img, target, index\n    \n#Custom Collate Function\ndef custom_collate_fn(batch):\n    data = torch.stack([item[0] for item in batch])\n    targets = torch.tensor([item[1] for item in batch])\n    indices = torch.tensor([item[2] for item in batch])\n    return data, targets, indices","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:50.474372Z","iopub.execute_input":"2024-08-17T09:45:50.474716Z","iopub.status.idle":"2024-08-17T09:45:50.487129Z","shell.execute_reply.started":"2024-08-17T09:45:50.474681Z","shell.execute_reply":"2024-08-17T09:45:50.486263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data and transform the dataset\ntrain_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_train)\nvalidation_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_test)\n\n# Create train and validation batch for training\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=100, shuffle=True, num_workers=2, collate_fn=custom_collate_fn)\nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=200, shuffle=False, num_workers=2, collate_fn=custom_collate_fn)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:50.488276Z","iopub.execute_input":"2024-08-17T09:45:50.488628Z","iopub.status.idle":"2024-08-17T09:45:59.071322Z","shell.execute_reply.started":"2024-08-17T09:45:50.488601Z","shell.execute_reply":"2024-08-17T09:45:59.070328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_data(img):\n    img = im_convert(img)\n    plt.imshow(img)\n    plt.show()\n    \n# We need to convert the images to numpy arrays as tensors are not compatible with matplotlib\ndef im_convert(tensor):\n    img = tensor.cpu().clone().detach().numpy()\n    img = img.transpose(1,2,0)\n    img = img*np.array(tuple(mean)) + np.array(tuple(std))\n    img = img.clip(0,1) #Clipping the size to print the images later\n    return img","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.072619Z","iopub.execute_input":"2024-08-17T09:45:59.072927Z","iopub.status.idle":"2024-08-17T09:45:59.079346Z","shell.execute_reply.started":"2024-08-17T09:45:59.072901Z","shell.execute_reply":"2024-08-17T09:45:59.078383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Visualization","metadata":{}},{"cell_type":"code","source":"# # Different classes in CIFAR10 dataset:\n# classes = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n# # Define an iterable on the data\n# data_iterable = iter(train_loader) # Converting our train_dataloader to iterable so that we can iterate through it.\n# images, labels = next(data_iterable) # Going from 1st batch of 100 images to the next batch\n# fig = plt.figure(figsize=(20,10))\n\n# # Let's plot 50 images from our train_dataset\n# for idx in np.arange(10):\n#     ax = fig.add_subplot(2, 5, idx+1, xticks=[], yticks=[])\n    \n#     # Note: imshow can't print tensor\n#     # Let's convert tensor image to numpy using im_convert function for imshow to print the image\n#     plt.imshow(im_convert(images[idx]))\n#     ax.set_title(classes[labels[idx].item()])","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.080814Z","iopub.execute_input":"2024-08-17T09:45:59.081255Z","iopub.status.idle":"2024-08-17T09:45:59.091242Z","shell.execute_reply.started":"2024-08-17T09:45:59.081220Z","shell.execute_reply":"2024-08-17T09:45:59.090444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Classifier**","metadata":{}},{"cell_type":"code","source":"# Cosine Normalization Classifier\nclass CosNorm_Classifier(nn.Module):\n    def __init__(self, in_dims, out_dims, scale=16, margin=0.5, init_std=0.001):\n        super(CosNorm_Classifier, self).__init__()\n        self.in_dims = in_dims\n        self.out_dims = out_dims\n        self.scale = scale\n        self.margin = margin\n        self.weight = Parameter(torch.Tensor(out_dims, in_dims).cuda())\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n\n    def forward(self, input, *args):\n        norm_x = torch.norm(input.clone(), 2, 1, keepdim=True)\n        ex = (norm_x / (1 + norm_x)) * (input / norm_x)\n        ew = self.weight / torch.norm(self.weight, 2, 1, keepdim=True)\n        return torch.mm(self.scale * ex, ew.t())\n    \ndef create_model(in_dims=512, out_dims=10):\n    print('Loading Cosine Norm Classifier.')\n    return CosNorm_Classifier(in_dims=in_dims, out_dims=out_dims)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.092352Z","iopub.execute_input":"2024-08-17T09:45:59.092724Z","iopub.status.idle":"2024-08-17T09:45:59.107184Z","shell.execute_reply.started":"2024-08-17T09:45:59.092698Z","shell.execute_reply":"2024-08-17T09:45:59.106364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_model(use_modulatedatt=False, use_fc=False, dropout=None, stage1_weights=False, dataset=None, test=False, *args):\n    \n#     print('Loading Scratch ResNet 10 Feature Model.')\n#     resnet10 = ResNet(BasicBlock, [1, 1, 1, 1], use_modulatedatt=use_modulatedatt, use_fc=use_fc, dropout=None)\n\n#     if not test:\n#         if stage1_weights:\n#             assert(dataset)\n#             print('Loading %s Stage 1 ResNet 10 Weights.' % dataset)\n#             resnet10 = init_weights(model=resnet10,\n#                                     weights_path='./logs/%s/stage1/final_model_checkpoint.pth' % dataset)\n#         else:\n#             print('No Pretrained Weights For Feature Model.')\n\n#     return resnet10","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.108447Z","iopub.execute_input":"2024-08-17T09:45:59.108766Z","iopub.status.idle":"2024-08-17T09:45:59.117754Z","shell.execute_reply.started":"2024-08-17T09:45:59.108729Z","shell.execute_reply":"2024-08-17T09:45:59.116870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Necessary functions for Triplet Loss","metadata":{}},{"cell_type":"markdown","source":"**Pairwise distances**","metadata":{}},{"cell_type":"code","source":"def pairwise_distance(features, squared=False):\n    pairwise_distances_squared = (\n        torch.sum(features ** 2, dim=1, keepdim=True) + \n        torch.sum(features.t() ** 2, dim=0, keepdim=True) - \n        2.0 * torch.matmul(features, features.t())\n    )\n    \n    pairwise_distances_squared = torch.max(pairwise_distances_squared, torch.tensor(0.0))\n    \n    error_mask = pairwise_distances_squared <= 0.0\n    \n    if squared:\n        pairwise_distances = pairwise_distances_squared\n    else:\n        pairwise_distances = torch.sqrt(pairwise_distances_squared + error_mask.float() * 1e-16)\n        \n    pairwise_distances = pairwise_distances * (error_mask == 0).float()\n    \n    num_data = features.shape[0]\n    mask_offdiagonals = torch.ones_like(pairwise_distances) - torch.eye(num_data, device=features.device)\n    pairwise_distances = pairwise_distances * mask_offdiagonals\n    return pairwise_distances","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.121106Z","iopub.execute_input":"2024-08-17T09:45:59.121478Z","iopub.status.idle":"2024-08-17T09:45:59.133441Z","shell.execute_reply.started":"2024-08-17T09:45:59.121451Z","shell.execute_reply":"2024-08-17T09:45:59.132478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Masked Maximum and Minimum**","metadata":{}},{"cell_type":"code","source":"def masked_maximum(data, mask, dim=1):\n    axis_minimums = torch.min(data, dim=dim, keepdim=True)[0]\n    masked_maximums = torch.max((data - axis_minimums)*mask.float(), dim=dim, keepdim=True)[0] + axis_minimums\n    return masked_maximums\n\ndef masked_minimum(data, mask, dim=1):\n    axis_maximums = torch.max(data, dim=dim, keepdim=True)[0]\n    masked_minimums = torch.min((data - axis_maximums)*mask.float(), dim=dim, keepdim=True)[0] + axis_maximums\n    return masked_minimums","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.134667Z","iopub.execute_input":"2024-08-17T09:45:59.134922Z","iopub.status.idle":"2024-08-17T09:45:59.146130Z","shell.execute_reply.started":"2024-08-17T09:45:59.134900Z","shell.execute_reply":"2024-08-17T09:45:59.145210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Semi-Hard Triplet Loss**","metadata":{}},{"cell_type":"code","source":"def triplet_loss(labels, embeddings, margin=1.0):\n    labels = labels.to(device)\n    embeddings = embeddings.to(device)\n    \n    pairwise_dist = torch.cdist(embeddings, embeddings, p=2)\n    \n    adjacency = labels.unsqueeze(1) == labels.unsqueeze(0)\n    adjacency_not = ~adjacency\n\n    batch_size = labels.size(0)\n\n    pdist_matrix_tile = pairwise_dist.repeat(batch_size, 1)\n    mask = adjacency_not.repeat(batch_size, 1) & (pdist_matrix_tile > pairwise_dist.t().reshape(-1, 1))\n\n    mask_final = mask.sum(dim=1, keepdim=True) > 0\n    mask_final = mask_final.view(batch_size, batch_size).t()\n\n    # Find the correct shape for negatives_outside\n    try:\n        negatives_outside = pdist_matrix_tile[mask].view(batch_size, batch_size - 1)\n    except RuntimeError:\n        negatives_outside = pdist_matrix_tile[mask]\n        negatives_outside = negatives_outside[:batch_size * (batch_size - 1)].view(batch_size, batch_size - 1)\n    \n    negatives_outside = negatives_outside.min(dim=1, keepdim=True)[0]\n\n    negatives_inside = pairwise_dist.clone()\n    negatives_inside[adjacency_not] = float('inf')\n    negatives_inside = negatives_inside.min(dim=1, keepdim=True)[0]\n\n    semi_hard_negatives = torch.where(mask_final, negatives_outside, negatives_inside)\n\n    loss = pairwise_dist - semi_hard_negatives + margin\n    loss[loss < 0] = 0\n\n    return loss.mean()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.147305Z","iopub.execute_input":"2024-08-17T09:45:59.147842Z","iopub.status.idle":"2024-08-17T09:45:59.163571Z","shell.execute_reply.started":"2024-08-17T09:45:59.147817Z","shell.execute_reply":"2024-08-17T09:45:59.162565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model**","metadata":{}},{"cell_type":"code","source":"class SimpleNet(nn.Module):\n    def __init__(self, num_classes=10):\n        super(SimpleNet, self).__init__()\n        # Replace with ResNet initialization\n        self.resnet10 = models.resnet18(pretrained=True)\n        self.resnet10.fc = nn.Identity() # Remove the final fully connected layer\n        self.classifier = CosNorm_Classifier(512, num_classes)\n        \n    def forward(self,x):\n        features = self.resnet10(x) # Assume ResNet returns features here\n#         print(f\"Output shape of ResNet: {x.shape}\") # Debug print\n        logits = self.classifier(features)\n        return logits\n    \nmodel = SimpleNet(num_classes = 10).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.164939Z","iopub.execute_input":"2024-08-17T09:45:59.165246Z","iopub.status.idle":"2024-08-17T09:45:59.973879Z","shell.execute_reply.started":"2024-08-17T09:45:59.165209Z","shell.execute_reply":"2024-08-17T09:45:59.973016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Base network creation function\n# class BaseNetwork(nn.Module):\n#     def __init__(self, embedding_size):\n#         super(BaseNetwork, self).__init__()\n#         self.flatten = nn.Flatten()\n#         self.fc1 = nn.Linear(3*32*32, 128)\n#         self.dropout1 = nn.Dropout(0.1)\n#         self.fc2 = nn.Linear(128, 128)\n#         self.dropout2 = nn.Dropout(0.1)\n#         self.fc3 = nn.Linear(128, embedding_size)\n        \n#     def forward(self, x):\n#         x = self.flatten(x)\n#         x = F.relu(self.fc1(x))\n#         x = self.dropout1(x)\n#         x = F.relu(self.fc2(x))\n#         x = self.dropout2(x)\n#         x = self.fc3(x)\n#         return x","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.975078Z","iopub.execute_input":"2024-08-17T09:45:59.975456Z","iopub.status.idle":"2024-08-17T09:45:59.980704Z","shell.execute_reply.started":"2024-08-17T09:45:59.975422Z","shell.execute_reply":"2024-08-17T09:45:59.979763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class FeatureExtractor(nn.Module):\n#     def __init__(self):\n#         super(FeatureExtractor, self).__init__()\n#         # Example feature extractor architecture (e.g., simple MLP or pre-trained model)\n#         self.resnet18 = models.resnet18(pretrained=True)\n#         self.resnet18.fc = nn.Identity() # Remove the last classification layer\n#         self.classifier = CosNorm_Classifier(512, num_classes)\n\n#     def forward(self, x):\n#         features = self.resnet(x)\n#         logits = self.classifier(features)\n#         return x\n    \n# # Initialize feature extractor\n# feature_extractor = FeatureExtractor().to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:45:59.981785Z","iopub.execute_input":"2024-08-17T09:45:59.982071Z","iopub.status.idle":"2024-08-17T09:45:59.990849Z","shell.execute_reply.started":"2024-08-17T09:45:59.982047Z","shell.execute_reply":"2024-08-17T09:45:59.989880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize model, Optimizer\n# embedded_size = 3*IMAGE_SIZE*IMAGE_SIZE\n# num_classes = 10\n# feat_dim = 2048 \n#model = create_model(use_modulatedatt=False, use_fc=False, dropout=None, stage1_weights=False, dataset='CIFAR10').to(device)\nmodel = SimpleNet(num_classes=10).to(device)\noptimizer = optim.Adam(model.parameters(), lr = 0.0001)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:46:20.361835Z","iopub.execute_input":"2024-08-17T09:46:20.362557Z","iopub.status.idle":"2024-08-17T09:46:20.626011Z","shell.execute_reply.started":"2024-08-17T09:46:20.362519Z","shell.execute_reply":"2024-08-17T09:46:20.624893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# centroids = torch.randn(num_classes, feat_dim).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:46:23.833783Z","iopub.execute_input":"2024-08-17T09:46:23.834150Z","iopub.status.idle":"2024-08-17T09:46:23.839740Z","shell.execute_reply.started":"2024-08-17T09:46:23.834121Z","shell.execute_reply":"2024-08-17T09:46:23.838469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get embeddings before training\nmodel.eval()\nembeddings_before_train = []\nlabels_before_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        data = data.to(device) #Flatten the input\n        target = target.to(device)\n        logits = model(data)\n        embeddings_before_train.append(logits.cpu().numpy())\n        labels_before_train.append(target.cpu().numpy())\n        \nembeddings_before_train = np.concatenate(embeddings_before_train)\nlabels_before_train = np.concatenate(labels_before_train)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:46:26.188980Z","iopub.execute_input":"2024-08-17T09:46:26.189829Z","iopub.status.idle":"2024-08-17T09:46:35.511398Z","shell.execute_reply.started":"2024-08-17T09:46:26.189786Z","shell.execute_reply":"2024-08-17T09:46:35.510357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Loop\nnum_epochs = 10\ntraining_loss = []\nmodel.train()\n# feature_extractor.train()\nfor epoch in range(num_epochs):\n    epoch_loss = 0\n    for data, target, _ in train_loader:\n        optimizer.zero_grad()\n        data = data.to(device) #Flatten the input\n        target = target.to(device) # Move target to device\n        \n#         features = feature_extractor(data)\n#          # Debugging: Print feature shape\n#         print(f\"Features shape: {features.shape}\")\n        \n        logits = model(data)\n        loss = triplet_loss(target, logits)\n        loss.backward()\n        optimizer.step()\n        epoch_loss += loss.item()\n    training_loss.append(epoch_loss / len(train_loader))\n        \n    print(f'Epoch {epoch + 1}/{num_epochs}, Loss: {epoch_loss / len(train_loader): .4f}')\n    \n    # Evaluate in validation set\n    model.eval()\n#     feature_extractor.eval()\n    all_targets = []\n    all_embeddings = []\n    all_predictions = []\n    start_time = time.time()\n    with torch.no_grad():\n        for data, target, index in validation_loader:\n            data = data.to(device)  # Flatten the input and move to device\n            target = target.to(device) # Move target to device\n            \n#             features = feature_extractor(data)\n            \n            logits = model(data)\n#             outputs = model(data)\n            predictions = torch.argmax(logits, dim=1)\n            all_targets.extend(target.cpu().numpy())\n            all_embeddings.extend(logits.cpu().numpy())\n            all_predictions.extend(predictions.cpu().numpy())\n    end_time = time.time()\n    testing_time = end_time - start_time\n    \n    # Calculate metrics\n    accuracy = accuracy_score(all_targets, all_predictions)\n    balanced_acc = balanced_accuracy_score(all_targets, all_predictions)\n    f1 = f1_score(all_targets, all_predictions, average='weighted')\n    recall = recall_score(all_targets, all_predictions, average='weighted')\n    precision = precision_score(all_targets, all_predictions, average='weighted')\n    \n    print(f'Epoch {epoch+1}, Accuracy: {accuracy:.4f}, Balanced Accuracy: {balanced_acc:.4f},'\n          f'F1 Score: {f1: .4f}, Recall: {recall: .4f}, Precision: {precision: .4f},'\n          f'Testing Time: {testing_time: .4f}s')","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:48:33.463932Z","iopub.execute_input":"2024-08-17T09:48:33.464667Z","iopub.status.idle":"2024-08-17T09:54:38.247433Z","shell.execute_reply.started":"2024-08-17T09:48:33.464629Z","shell.execute_reply":"2024-08-17T09:54:38.246353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After training, plot ROC Curve\nmodel.eval()\nwith torch.no_grad():\n    # Convert lists to numpy arrays\n    all_targets = np.array(all_targets)\n    all_embeddings = np.array(all_embeddings)\n    \n    # Binarize the labels for multi-class ROC computation\n    y_bin = label_binarize(all_targets, classes=np.arange(10))  # Adjust `classes` based on the number of classes\n    y_score = np.array(all_embeddings)  # Adjust this if you need to obtain scores from embeddings\n    \n    # Plot ROC curves for each class\n    plt.figure(figsize=(10, 8))\n    for i in range(10):  # Assuming 10 classes\n        fpr, tpr, _ = roc_curve(y_bin[:, i], y_score[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label=f'Class {i} (area = {roc_auc:.2f})')\n    \n    plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic for Multi-Class')\n    plt.legend(loc='lower right')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:55:02.873930Z","iopub.execute_input":"2024-08-17T09:55:02.874338Z","iopub.status.idle":"2024-08-17T09:55:03.490974Z","shell.execute_reply.started":"2024-08-17T09:55:02.874283Z","shell.execute_reply":"2024-08-17T09:55:03.489948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get embeddings after train\nmodel.eval()\nembeddings_after_train = []\nlabels_after_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        data = data.to(device) #Flatten the input\n        target = target.to(device)\n        \n        logits = model(data)\n        embeddings_after_train.append(logits.cpu().numpy())\n        labels_after_train.append(target.cpu().numpy())\n\nembeddings_after_train = np.concatenate(embeddings_after_train)\nlabels_after_train = np.concatenate(labels_after_train)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:55:40.377672Z","iopub.execute_input":"2024-08-17T09:55:40.378467Z","iopub.status.idle":"2024-08-17T09:55:48.972667Z","shell.execute_reply.started":"2024-08-17T09:55:40.378433Z","shell.execute_reply":"2024-08-17T09:55:48.971581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the effect of embeddings using PCA\nnum_components=2\nstep = 7\n\npca = PCA(n_components=num_components)\ndecomposed_before_train = pca.fit_transform(embeddings_before_train)\ndecomposed_after_train = pca.fit_transform(embeddings_after_train)\ntest_class_labels = np.unique(labels_after_train)\n\nfig = plt.figure(figsize=(16, 8))\nfor label in test_class_labels:\n    decomposed_before_class = decomposed_before_train[labels_before_train == label]\n    decomposed_after_class = decomposed_after_train[labels_after_train == label]\n    \n    plt.subplot(1, 2, 1)\n    plt.scatter(decomposed_before_class[:, 0], decomposed_before_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('Before Training (embeddings)')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.scatter(decomposed_after_class[:, 0], decomposed_after_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('After {num_epochs} Epochs (embeddings)')\n    plt.legend()\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:55:51.461608Z","iopub.execute_input":"2024-08-17T09:55:51.462013Z","iopub.status.idle":"2024-08-17T09:55:54.159587Z","shell.execute_reply.started":"2024-08-17T09:55:51.461979Z","shell.execute_reply":"2024-08-17T09:55:54.158619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the trained model\ntorch.save(model.state_dict(), 'cifar10_base_network.pth')","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:58:25.461848Z","iopub.execute_input":"2024-08-17T09:58:25.462255Z","iopub.status.idle":"2024-08-17T09:58:25.557271Z","shell.execute_reply.started":"2024-08-17T09:58:25.462226Z","shell.execute_reply":"2024-08-17T09:58:25.556447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the training data\nplt.figure(figsize=(8,8))\nplt.plot(training_loss, label='Training Loss')\nplt.legend()\nplt.title('Training Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-17T09:58:26.284817Z","iopub.execute_input":"2024-08-17T09:58:26.285765Z","iopub.status.idle":"2024-08-17T09:58:26.511167Z","shell.execute_reply.started":"2024-08-17T09:58:26.285730Z","shell.execute_reply":"2024-08-17T09:58:26.510124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}