{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle\nfrom PIL import Image\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport os\nimport torch.nn.functional as F\nimport requests\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport seaborn as sn\ntorch.manual_seed(1) #Set manual seed\nfrom sklearn import preprocessing\nfrom torch.utils.data import DataLoader, Dataset, Subset\nimport torch.optim as optim\nfrom sklearn.metrics import roc_auc_score, average_precision_score, balanced_accuracy_score, f1_score, accuracy_score, recall_score, precision_score, roc_curve, auc\nfrom time import time\nfrom sklearn.preprocessing import label_binarize\nimport torchvision.models as models\n# from torchvision.models import resnet10\\\nfrom sklearn.decomposition import PCA\nfrom torchvision.models.resnet import ResNet, Bottleneck\n\n#Define GPU device\nmps_device = torch.device(\"mps\")\n\nimport torch\nimport math\nimport torch.nn as nn\nfrom torch.nn.parameter import Parameter\nimport time\n\nimport pdb\n\nimport math\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from layers.ModulatedAttLayer import ModulatedAttLayer","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:25:31.580950Z","iopub.execute_input":"2024-08-17T10:25:31.581346Z","iopub.status.idle":"2024-08-17T10:25:31.593180Z","shell.execute_reply.started":"2024-08-17T10:25:31.581317Z","shell.execute_reply":"2024-08-17T10:25:31.592199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:24:04.426190Z","iopub.execute_input":"2024-08-17T10:24:04.426635Z","iopub.status.idle":"2024-08-17T10:24:04.461138Z","shell.execute_reply.started":"2024-08-17T10:24:04.426609Z","shell.execute_reply":"2024-08-17T10:24:04.460001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = 32\nmean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261]\n# These values are mostly used by researchers as found to very useful in fast convergence\n\n# We can try various transformation for good generalization of model.\ncomposed_train = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), # Resize the image in a 32 x 32 shape\n                                     transforms.RandomRotation(20), # Randomly rotate some images by 20 degrees\n                                     transforms.RandomHorizontalFlip(0.1), #Randomly horizontal flip the images\n                                     transforms.ColorJitter(brightness = 0.1, #Randomly adjust color jitter of the images\n                                                            contrast = 0.1,\n                                                            saturation = 0.1),\n                                     transforms.RandomAdjustSharpness(sharpness_factor = 2,\n                                                                      p = 0.1), # Randomly adjust sharpness\n                                     transforms.ToTensor(), # Converting image to tensor\n                                     transforms.Normalize(mean, std), # Normalizing with standard mean and standard deviation\n                                     transforms.RandomErasing(p = 0.75, scale = (0.02, 0.1), value = 1.0, inplace = False)\n                                    ])\n\ncomposed_test = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize(mean, std)\n                                   ])","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:24:04.462615Z","iopub.execute_input":"2024-08-17T10:24:04.462993Z","iopub.status.idle":"2024-08-17T10:24:04.492521Z","shell.execute_reply.started":"2024-08-17T10:24:04.462959Z","shell.execute_reply":"2024-08-17T10:24:04.491525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCIFAR10(datasets.CIFAR10):\n    def __getitem__(self, index):\n        img, target = super().__getitem__(index)\n        return img, target, index\n    \n#Custom Collate Function\ndef custom_collate_fn(batch):\n    data = torch.stack([item[0] for item in batch])\n    targets = torch.tensor([item[1] for item in batch])\n    indices = torch.tensor([item[2] for item in batch])\n    return data, targets, indices","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:24:04.495161Z","iopub.execute_input":"2024-08-17T10:24:04.495488Z","iopub.status.idle":"2024-08-17T10:24:04.508599Z","shell.execute_reply.started":"2024-08-17T10:24:04.495451Z","shell.execute_reply":"2024-08-17T10:24:04.507625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data and transform the dataset\ntrain_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_train)\nvalidation_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_test)\n\n# Create train and validation batch for training\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=100, shuffle=True, num_workers=2, collate_fn=custom_collate_fn)\nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=200, shuffle=False, num_workers=2, collate_fn=custom_collate_fn)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:24:04.509715Z","iopub.execute_input":"2024-08-17T10:24:04.510016Z","iopub.status.idle":"2024-08-17T10:24:14.167361Z","shell.execute_reply.started":"2024-08-17T10:24:04.509991Z","shell.execute_reply":"2024-08-17T10:24:14.166361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_data(img):\n    img = im_convert(img)\n    plt.imshow(img)\n    plt.show()\n    \n# We need to convert the images to numpy arrays as tensors are not compatible with matplotlib\ndef im_convert(tensor):\n    img = tensor.cpu().clone().detach().numpy()\n    img = img.transpose(1,2,0)\n    img = img*np.array(tuple(mean)) + np.array(tuple(std))\n    img = img.clip(0,1) #Clipping the size to print the images later\n    return img","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:24:14.168635Z","iopub.execute_input":"2024-08-17T10:24:14.168946Z","iopub.status.idle":"2024-08-17T10:24:14.177363Z","shell.execute_reply.started":"2024-08-17T10:24:14.168921Z","shell.execute_reply":"2024-08-17T10:24:14.176377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Visualization","metadata":{}},{"cell_type":"code","source":"# # Different classes in CIFAR10 dataset:\n# classes = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n# # Define an iterable on the data\n# data_iterable = iter(train_loader) # Converting our train_dataloader to iterable so that we can iterate through it.\n# images, labels = next(data_iterable) # Going from 1st batch of 100 images to the next batch\n# fig = plt.figure(figsize=(20,10))\n\n# # Let's plot 50 images from our train_dataset\n# for idx in np.arange(10):\n#     ax = fig.add_subplot(2, 5, idx+1, xticks=[], yticks=[])\n    \n#     # Note: imshow can't print tensor\n#     # Let's convert tensor image to numpy using im_convert function for imshow to print the image\n#     plt.imshow(im_convert(images[idx]))\n#     ax.set_title(classes[labels[idx].item()])","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:24:14.179096Z","iopub.execute_input":"2024-08-17T10:24:14.179526Z","iopub.status.idle":"2024-08-17T10:24:14.191033Z","shell.execute_reply.started":"2024-08-17T10:24:14.179493Z","shell.execute_reply":"2024-08-17T10:24:14.189984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Classifier**","metadata":{}},{"cell_type":"code","source":"# Cosine Normalization Classifier\nclass CosNorm_Classifier(nn.Module):\n    def __init__(self, in_dims, out_dims, scale=10, margin=0.5, init_std=0.001):\n        super(CosNorm_Classifier, self).__init__()\n        self.in_dims = in_dims\n        self.out_dims = out_dims\n        self.scale = scale\n        self.margin = margin\n        self.weight = Parameter(torch.Tensor(out_dims, in_dims).cuda())\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n\n    def forward(self, input, *args):\n        norm_x = torch.norm(input.clone(), 2, 1, keepdim=True)\n        ex = (norm_x / (1 + norm_x)) * (input / norm_x)\n        ew = self.weight / torch.norm(self.weight, 2, 1, keepdim=True)\n        return torch.mm(self.scale * ex, ew.t())\n    \n# def create_model(in_dims=512, out_dims=10):\n#     print('Loading Cosine Norm Classifier.')\n#     return CosNorm_Classifier(in_dims=in_dims, out_dims=out_dims)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:52.847765Z","iopub.execute_input":"2024-08-17T10:43:52.848607Z","iopub.status.idle":"2024-08-17T10:43:52.857609Z","shell.execute_reply.started":"2024-08-17T10:43:52.848547Z","shell.execute_reply":"2024-08-17T10:43:52.856619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_model(use_modulatedatt=False, use_fc=False, dropout=None, stage1_weights=False, dataset=None, test=False, *args):\n    \n#     print('Loading Scratch ResNet 10 Feature Model.')\n#     resnet10 = ResNet(BasicBlock, [1, 1, 1, 1], use_modulatedatt=use_modulatedatt, use_fc=use_fc, dropout=None)\n\n#     if not test:\n#         if stage1_weights:\n#             assert(dataset)\n#             print('Loading %s Stage 1 ResNet 10 Weights.' % dataset)\n#             resnet10 = init_weights(model=resnet10,\n#                                     weights_path='./logs/%s/stage1/final_model_checkpoint.pth' % dataset)\n#         else:\n#             print('No Pretrained Weights For Feature Model.')\n\n#     return resnet10","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:53.245468Z","iopub.execute_input":"2024-08-17T10:43:53.246168Z","iopub.status.idle":"2024-08-17T10:43:53.250853Z","shell.execute_reply.started":"2024-08-17T10:43:53.246133Z","shell.execute_reply":"2024-08-17T10:43:53.249720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Necessary functions for Triplet Loss","metadata":{}},{"cell_type":"markdown","source":"**Pairwise distances**","metadata":{}},{"cell_type":"code","source":"def pairwise_distance(features, squared=False):\n    pairwise_distances_squared = (\n        torch.sum(features ** 2, dim=1, keepdim=True) + \n        torch.sum(features.t() ** 2, dim=0, keepdim=True) - \n        2.0 * torch.matmul(features, features.t())\n    )\n    \n    pairwise_distances_squared = torch.max(pairwise_distances_squared, torch.tensor(0.0))\n    \n    error_mask = pairwise_distances_squared <= 0.0\n    \n    if squared:\n        pairwise_distances = pairwise_distances_squared\n    else:\n        pairwise_distances = torch.sqrt(pairwise_distances_squared + error_mask.float() * 1e-16)\n        \n    pairwise_distances = pairwise_distances * (error_mask == 0).float()\n    \n    num_data = features.shape[0]\n    mask_offdiagonals = torch.ones_like(pairwise_distances) - torch.eye(num_data, device=features.device)\n    pairwise_distances = pairwise_distances * mask_offdiagonals\n    return pairwise_distances","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:54.308705Z","iopub.execute_input":"2024-08-17T10:43:54.309112Z","iopub.status.idle":"2024-08-17T10:43:54.318121Z","shell.execute_reply.started":"2024-08-17T10:43:54.309072Z","shell.execute_reply":"2024-08-17T10:43:54.316991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Masked Maximum and Minimum**","metadata":{}},{"cell_type":"code","source":"def masked_maximum(data, mask, dim=1):\n    axis_minimums = torch.min(data, dim=dim, keepdim=True)[0]\n    masked_maximums = torch.max((data - axis_minimums)*mask.float(), dim=dim, keepdim=True)[0] + axis_minimums\n    return masked_maximums\n\ndef masked_minimum(data, mask, dim=1):\n    axis_maximums = torch.max(data, dim=dim, keepdim=True)[0]\n    masked_minimums = torch.min((data - axis_maximums)*mask.float(), dim=dim, keepdim=True)[0] + axis_maximums\n    return masked_minimums","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:55.142633Z","iopub.execute_input":"2024-08-17T10:43:55.143618Z","iopub.status.idle":"2024-08-17T10:43:55.150539Z","shell.execute_reply.started":"2024-08-17T10:43:55.143556Z","shell.execute_reply":"2024-08-17T10:43:55.149628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Semi-Hard Triplet Loss**","metadata":{}},{"cell_type":"code","source":"def triplet_loss(labels, embeddings, margin=1.0):\n    labels = labels.to(device)\n    embeddings = embeddings.to(device)\n    \n    pairwise_dist = torch.cdist(embeddings, embeddings, p=2)\n    \n    adjacency = labels.unsqueeze(1) == labels.unsqueeze(0)\n    adjacency_not = ~adjacency\n\n    batch_size = labels.size(0)\n\n    pdist_matrix_tile = pairwise_dist.repeat(batch_size, 1)\n    mask = adjacency_not.repeat(batch_size, 1) & (pdist_matrix_tile > pairwise_dist.t().reshape(-1, 1))\n\n    mask_final = mask.sum(dim=1, keepdim=True) > 0\n    mask_final = mask_final.view(batch_size, batch_size).t()\n\n    # Find the correct shape for negatives_outside\n    try:\n        negatives_outside = pdist_matrix_tile[mask].view(batch_size, batch_size - 1)\n    except RuntimeError:\n        negatives_outside = pdist_matrix_tile[mask]\n        negatives_outside = negatives_outside[:batch_size * (batch_size - 1)].view(batch_size, batch_size - 1)\n    \n    negatives_outside = negatives_outside.min(dim=1, keepdim=True)[0]\n\n    negatives_inside = pairwise_dist.clone()\n    negatives_inside[adjacency_not] = float('inf')\n    negatives_inside = negatives_inside.min(dim=1, keepdim=True)[0]\n\n    semi_hard_negatives = torch.where(mask_final, negatives_outside, negatives_inside)\n\n    loss = pairwise_dist - semi_hard_negatives + margin\n    loss[loss < 0] = 0\n\n    return loss.mean()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:56.926063Z","iopub.execute_input":"2024-08-17T10:43:56.926431Z","iopub.status.idle":"2024-08-17T10:43:56.936473Z","shell.execute_reply.started":"2024-08-17T10:43:56.926400Z","shell.execute_reply":"2024-08-17T10:43:56.935531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model**","metadata":{}},{"cell_type":"code","source":"class SimpleNet(nn.Module):\n    def __init__(self, num_classes):\n        super(SimpleNet, self).__init__()\n        # Replace with ResNet initialization\n        self.feature_extractor = FeatureExtractor(use_fc=True, dropout=0.5)\n        self.classifier = CosNorm_Classifier(1000, num_classes) #2048 is the output dimension\n        \n    def forward(self,x):\n        features = self.feature_extractor(x) \n#         print(f\"Output shape of ResNet: {x.shape}\") # Debug print\n        logits = self.classifier(features)\n        return logits","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:58.242788Z","iopub.execute_input":"2024-08-17T10:43:58.243889Z","iopub.status.idle":"2024-08-17T10:43:58.249715Z","shell.execute_reply.started":"2024-08-17T10:43:58.243852Z","shell.execute_reply":"2024-08-17T10:43:58.248743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Base network creation function\n# class BaseNetwork(nn.Module):\n#     def __init__(self, embedding_size):\n#         super(BaseNetwork, self).__init__()\n#         self.flatten = nn.Flatten()\n#         self.fc1 = nn.Linear(3*32*32, 128)\n#         self.dropout1 = nn.Dropout(0.1)\n#         self.fc2 = nn.Linear(128, 128)\n#         self.dropout2 = nn.Dropout(0.1)\n#         self.fc3 = nn.Linear(128, embedding_size)\n        \n#     def forward(self, x):\n#         x = self.flatten(x)\n#         x = F.relu(self.fc1(x))\n#         x = self.dropout1(x)\n#         x = F.relu(self.fc2(x))\n#         x = self.dropout2(x)\n#         x = self.fc3(x)\n#         return x","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:43:59.853493Z","iopub.execute_input":"2024-08-17T10:43:59.854369Z","iopub.status.idle":"2024-08-17T10:43:59.858769Z","shell.execute_reply.started":"2024-08-17T10:43:59.854334Z","shell.execute_reply":"2024-08-17T10:43:59.857752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.models as models\n\nclass CustomResNet152(models.ResNet):\n    def __init__(self, use_fc=False, dropout=None):\n        super(CustomResNet152, self).__init__(block=models.resnet.Bottleneck, layers=[3, 8, 36, 3])\n        # Add custom layers or modifications here\n        if use_fc:\n            self.fc = nn.Sequential(\n                nn.Dropout(dropout) if dropout else nn.Identity(),\n                nn.Linear(2048, 1000)  # Example fully connected layer\n            )\n    \n    def forward(self, x):\n        return super(CustomResNet152, self).forward(x)\n\nclass FeatureExtractor(nn.Module):\n    def __init__(self, use_fc=False, dropout=None, stage1_weights=False, dataset=None, caffe=False, test=False):\n        super(FeatureExtractor, self).__init__()\n        print('Loading Custom ResNet 152 Feature Model.')\n        self.resnet152 = CustomResNet152(use_fc=use_fc, dropout=dropout)\n\n        if not test:\n            # If neither caffe nor stage1_weights is used, this condition can be removed or adjusted\n            if caffe == stage1_weights:\n                print(\"Warning: Both 'caffe' and 'stage1_weights' are set to the same value. Continuing without pre-trained weights.\")\n            else:\n                if caffe:\n                    print('Loading Caffe Pretrained ResNet 152 Weights.')\n                    self.resnet152 = init_weights(model=self.resnet152, weights_path='./logs/caffe_resnet152.pth', caffe=True)\n                elif stage1_weights:\n                    assert dataset\n                    print(f'Loading {dataset} Stage 1 ResNet 152 Weights.')\n                    self.resnet152 = init_weights(model=self.resnet152, weights_path=f'./logs/{dataset}/stage1/final_model_checkpoint.pth')\n                else:\n                    print('No Pretrained Weights For Feature Model.')\n\n    def forward(self, x):\n        return self.resnet152(x)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:44:00.450131Z","iopub.execute_input":"2024-08-17T10:44:00.450981Z","iopub.status.idle":"2024-08-17T10:44:00.461586Z","shell.execute_reply.started":"2024-08-17T10:44:00.450946Z","shell.execute_reply":"2024-08-17T10:44:00.460680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize model, Optimizer\n# embedded_size = 3*IMAGE_SIZE*IMAGE_SIZE\n# num_classes = 10\n# feat_dim = 2048 \n#model = create_model(use_modulatedatt=False, use_fc=False, dropout=None, stage1_weights=False, dataset='CIFAR10').to(device)\nmodel = SimpleNet(num_classes=10).to(device)\nfeature_extractor = FeatureExtractor().to(device)\noptimizer = optim.Adam(list(model.parameters()), lr = 0.0001)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:44:02.961907Z","iopub.execute_input":"2024-08-17T10:44:02.962578Z","iopub.status.idle":"2024-08-17T10:44:05.378156Z","shell.execute_reply.started":"2024-08-17T10:44:02.962532Z","shell.execute_reply":"2024-08-17T10:44:05.377050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# centroids = torch.randn(num_classes, feat_dim).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:44:07.165298Z","iopub.execute_input":"2024-08-17T10:44:07.166144Z","iopub.status.idle":"2024-08-17T10:44:07.169892Z","shell.execute_reply.started":"2024-08-17T10:44:07.166110Z","shell.execute_reply":"2024-08-17T10:44:07.168971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get embeddings before training\nmodel.eval()\nembeddings_before_train = []\nlabels_before_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        data = data.to(device) #Flatten the input\n        target = target.to(device)\n        features = model.feature_extractor(data)  # Extract features from ResNet152\n        logits = model.classifier(features)\n        embeddings_before_train.append(logits.cpu().numpy())\n        labels_before_train.append(target.cpu().numpy())\n        \nembeddings_before_train = np.concatenate(embeddings_before_train)\nlabels_before_train = np.concatenate(labels_before_train)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:44:08.031007Z","iopub.execute_input":"2024-08-17T10:44:08.031900Z","iopub.status.idle":"2024-08-17T10:44:24.748338Z","shell.execute_reply.started":"2024-08-17T10:44:08.031867Z","shell.execute_reply":"2024-08-17T10:44:24.747001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Loop\nnum_epochs = 10\ntraining_loss = []\nmodel.train()\n# feature_extractor.train()\nfor epoch in range(num_epochs):\n    epoch_loss = 0\n    for data, target, _ in train_loader:\n        optimizer.zero_grad()\n        data = data.to(device) #Flatten the input\n        target = target.to(device) # Move target to device\n        \n        features = model.feature_extractor(data)\n#          # Debugging: Print feature shape\n#         print(f\"Features shape: {features.shape}\")\n        \n        logits = model.classifier(features)\n        \n        loss = triplet_loss(target, logits)\n        loss.backward()\n        optimizer.step()\n        epoch_loss += loss.item()\n    training_loss.append(epoch_loss / len(train_loader))\n        \n    print(f'Epoch {epoch + 1}/{num_epochs}, Loss: {epoch_loss / len(train_loader): .4f}')\n    \n    # Evaluate in validation set\n    model.eval()\n#     feature_extractor.eval()\n    all_targets = []\n    all_embeddings = []\n    all_predictions = []\n    start_time = time.time()\n    with torch.no_grad():\n        for data, target, index in validation_loader:\n            data = data.to(device)  # Flatten the input and move to device\n            target = target.to(device) # Move target to device\n            \n            features = model.feature_extractor(data)\n            \n            logits = model.classifier(features)\n#             outputs = model(data)\n            predictions = torch.argmax(logits, dim=1)\n            all_targets.extend(target.cpu().numpy())\n            all_embeddings.extend(logits.cpu().numpy())\n            all_predictions.extend(predictions.cpu().numpy())\n    end_time = time.time()\n    testing_time = end_time - start_time\n    \n    # Calculate metrics\n    accuracy = accuracy_score(all_targets, all_predictions)\n    balanced_acc = balanced_accuracy_score(all_targets, all_predictions)\n    f1 = f1_score(all_targets, all_predictions, average='weighted')\n    recall = recall_score(all_targets, all_predictions, average='weighted')\n    precision = precision_score(all_targets, all_predictions, average='weighted')\n    \n    print(f'Epoch {epoch+1}, Accuracy: {accuracy:.4f}, Balanced Accuracy: {balanced_acc:.4f},'\n          f'F1 Score: {f1: .4f}, Recall: {recall: .4f}, Precision: {precision: .4f},'\n          f'Testing Time: {testing_time: .4f}s')","metadata":{"execution":{"iopub.status.busy":"2024-08-17T10:44:27.862983Z","iopub.execute_input":"2024-08-17T10:44:27.863467Z","iopub.status.idle":"2024-08-17T11:01:38.329124Z","shell.execute_reply.started":"2024-08-17T10:44:27.863387Z","shell.execute_reply":"2024-08-17T11:01:38.327624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After training, plot ROC Curve\nmodel.eval()\nwith torch.no_grad():\n    # Convert lists to numpy arrays\n    all_targets = np.array(all_targets)\n    all_embeddings = np.array(all_embeddings)\n    \n    # Binarize the labels for multi-class ROC computation\n    y_bin = label_binarize(all_targets, classes=np.arange(10))  # Adjust `classes` based on the number of classes\n    y_score = np.array(all_embeddings)  # Adjust this if you need to obtain scores from embeddings\n    \n    # Plot ROC curves for each class\n    plt.figure(figsize=(10, 8))\n    for i in range(10):  # Assuming 10 classes\n        fpr, tpr, _ = roc_curve(y_bin[:, i], y_score[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label=f'Class {i} (area = {roc_auc:.2f})')\n    \n    plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic for Multi-Class')\n    plt.legend(loc='lower right')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-17T11:01:38.331786Z","iopub.execute_input":"2024-08-17T11:01:38.332707Z","iopub.status.idle":"2024-08-17T11:01:39.000147Z","shell.execute_reply.started":"2024-08-17T11:01:38.332658Z","shell.execute_reply":"2024-08-17T11:01:38.999227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get embeddings after train\nmodel.eval()\nembeddings_after_train = []\nlabels_after_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        data = data.to(device) #Flatten the input\n        target = target.to(device)\n        \n        features = model.feature_extractor(data)  # Extract features from ResNet152\n        logits = model.classifier(features)\n        embeddings_after_train.append(logits.cpu().numpy())\n        labels_after_train.append(target.cpu().numpy())\n\nembeddings_after_train = np.concatenate(embeddings_after_train)\nlabels_after_train = np.concatenate(labels_after_train)","metadata":{"execution":{"iopub.status.busy":"2024-08-17T11:01:39.001297Z","iopub.execute_input":"2024-08-17T11:01:39.001561Z","iopub.status.idle":"2024-08-17T11:01:55.785631Z","shell.execute_reply.started":"2024-08-17T11:01:39.001538Z","shell.execute_reply":"2024-08-17T11:01:55.784283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the effect of embeddings using PCA\nnum_components=2\nstep = 7\n\npca = PCA(n_components=num_components)\ndecomposed_before_train = pca.fit_transform(embeddings_before_train)\ndecomposed_after_train = pca.fit_transform(embeddings_after_train)\ntest_class_labels = np.unique(labels_after_train)\n\nfig = plt.figure(figsize=(16, 8))\nfor label in test_class_labels:\n    decomposed_before_class = decomposed_before_train[labels_before_train == label]\n    decomposed_after_class = decomposed_after_train[labels_after_train == label]\n    \n    plt.subplot(1, 2, 1)\n    plt.scatter(decomposed_before_class[:, 0], decomposed_before_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('Before Training (embeddings)')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.scatter(decomposed_after_class[:, 0], decomposed_after_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('After {num_epochs} Epochs (embeddings)')\n    plt.legend()\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-17T11:01:55.788293Z","iopub.execute_input":"2024-08-17T11:01:55.788666Z","iopub.status.idle":"2024-08-17T11:01:58.451994Z","shell.execute_reply.started":"2024-08-17T11:01:55.788632Z","shell.execute_reply":"2024-08-17T11:01:58.450982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the trained model\ntorch.save(model.state_dict(), 'cifar10_base_network.pth')","metadata":{"execution":{"iopub.status.busy":"2024-08-17T11:01:58.453208Z","iopub.execute_input":"2024-08-17T11:01:58.453548Z","iopub.status.idle":"2024-08-17T11:01:58.879881Z","shell.execute_reply.started":"2024-08-17T11:01:58.453521Z","shell.execute_reply":"2024-08-17T11:01:58.878997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the training data\nplt.figure(figsize=(8,8))\nplt.plot(training_loss, label='Training Loss')\nplt.legend()\nplt.title('Training Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-17T11:01:58.881456Z","iopub.execute_input":"2024-08-17T11:01:58.881774Z","iopub.status.idle":"2024-08-17T11:01:59.170749Z","shell.execute_reply.started":"2024-08-17T11:01:58.881749Z","shell.execute_reply":"2024-08-17T11:01:59.169732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}