{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":396.392444,"end_time":"2024-08-05T10:02:21.404737","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-05T09:55:45.012293","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle\nfrom PIL import Image\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport os\nimport torch.nn.functional as F\nimport requests\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport seaborn as sn\ntorch.manual_seed(1) #Set manual seed\nfrom sklearn import preprocessing\nfrom torch.utils.data import DataLoader, Dataset, Subset\nimport torch.optim as optim\nfrom sklearn.metrics import roc_auc_score, average_precision_score, balanced_accuracy_score, f1_score, accuracy_score, recall_score, precision_score, roc_curve, auc\nfrom time import time\nfrom sklearn.preprocessing import label_binarize\nimport torchvision.models as models\n# from torchvision.models import resnet10\\\nfrom sklearn.decomposition import PCA\n\n#Define GPU device\nmps_device = torch.device(\"mps\")\n\nimport torch\nimport math\nimport torch.nn as nn\nfrom torch.nn.parameter import Parameter\nimport time\n\nimport pdb\n\nimport math\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from layers.ModulatedAttLayer import ModulatedAttLayer","metadata":{"papermill":{"duration":7.244747,"end_time":"2024-08-05T09:55:55.019509","exception":false,"start_time":"2024-08-05T09:55:47.774762","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:31.078741Z","iopub.execute_input":"2024-08-05T15:11:31.079109Z","iopub.status.idle":"2024-08-05T15:11:37.532858Z","shell.execute_reply.started":"2024-08-05T15:11:31.079077Z","shell.execute_reply":"2024-08-05T15:11:37.531789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation","metadata":{"papermill":{"duration":0.006458,"end_time":"2024-08-05T09:55:55.032937","exception":false,"start_time":"2024-08-05T09:55:55.026479","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IMAGE_SIZE = 32\nmean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261]\n# These values are mostly used by researchers as found to very useful in fast convergence\n\n# We can try various transformation for good generalization of model.\ncomposed_train = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), # Resize the image in a 32 x 32 shape\n                                     transforms.RandomRotation(20), # Randomly rotate some images by 20 degrees\n                                     transforms.RandomHorizontalFlip(0.1), #Randomly horizontal flip the images\n                                     transforms.ColorJitter(brightness = 0.1, #Randomly adjust color jitter of the images\n                                                            contrast = 0.1,\n                                                            saturation = 0.1),\n                                     transforms.RandomAdjustSharpness(sharpness_factor = 2,\n                                                                      p = 0.1), # Randomly adjust sharpness\n                                     transforms.ToTensor(), # Converting image to tensor\n                                     transforms.Normalize(mean, std), # Normalizing with standard mean and standard deviation\n                                     transforms.RandomErasing(p = 0.75, scale = (0.02, 0.1), value = 1.0, inplace = False)\n                                    ])\n\ncomposed_test = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize(mean, std)\n                                   ])","metadata":{"papermill":{"duration":0.018018,"end_time":"2024-08-05T09:55:55.057496","exception":false,"start_time":"2024-08-05T09:55:55.039478","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:37.534608Z","iopub.execute_input":"2024-08-05T15:11:37.535084Z","iopub.status.idle":"2024-08-05T15:11:37.544182Z","shell.execute_reply.started":"2024-08-05T15:11:37.535054Z","shell.execute_reply":"2024-08-05T15:11:37.543086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCIFAR10(datasets.CIFAR10):\n    def __getitem__(self, index):\n        img, target = super().__getitem__(index)\n        return img, target, index\n    \n#Custom Collate Function\ndef custom_collate_fn(batch):\n    data = torch.stack([item[0] for item in batch])\n    targets = torch.tensor([item[1] for item in batch])\n    indices = torch.tensor([item[2] for item in batch])\n    return data, targets, indices","metadata":{"papermill":{"duration":0.015294,"end_time":"2024-08-05T09:55:55.079447","exception":false,"start_time":"2024-08-05T09:55:55.064153","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:37.545505Z","iopub.execute_input":"2024-08-05T15:11:37.545807Z","iopub.status.idle":"2024-08-05T15:11:37.584677Z","shell.execute_reply.started":"2024-08-05T15:11:37.545782Z","shell.execute_reply":"2024-08-05T15:11:37.583586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data and transform the dataset\ntrain_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_train)\nvalidation_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_test)\n\n# Create train and validation batch for training\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=100, shuffle=True, num_workers=2, collate_fn=custom_collate_fn)\nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=200, shuffle=False, num_workers=2, collate_fn=custom_collate_fn)","metadata":{"papermill":{"duration":15.593755,"end_time":"2024-08-05T09:56:10.679708","exception":false,"start_time":"2024-08-05T09:55:55.085953","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:37.587092Z","iopub.execute_input":"2024-08-05T15:11:37.587451Z","iopub.status.idle":"2024-08-05T15:11:52.839103Z","shell.execute_reply.started":"2024-08-05T15:11:37.587423Z","shell.execute_reply":"2024-08-05T15:11:52.838282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_data(img):\n    img = im_convert(img)\n    plt.imshow(img)\n    plt.show()\n    \n# We need to convert the images to numpy arrays as tensors are not compatible with matplotlib\ndef im_convert(tensor):\n    img = tensor.cpu().clone().detach().numpy()\n    img = img.transpose(1,2,0)\n    img = img*np.array(tuple(mean)) + np.array(tuple(std))\n    img = img.clip(0,1) #Clipping the size to print the images later\n    return img","metadata":{"papermill":{"duration":0.021621,"end_time":"2024-08-05T09:56:10.714023","exception":false,"start_time":"2024-08-05T09:56:10.692402","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:52.840269Z","iopub.execute_input":"2024-08-05T15:11:52.840578Z","iopub.status.idle":"2024-08-05T15:11:52.847418Z","shell.execute_reply.started":"2024-08-05T15:11:52.840552Z","shell.execute_reply":"2024-08-05T15:11:52.846295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Visualization","metadata":{"papermill":{"duration":0.011741,"end_time":"2024-08-05T09:56:10.738070","exception":false,"start_time":"2024-08-05T09:56:10.726329","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # Different classes in CIFAR10 dataset:\n# classes = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n# # Define an iterable on the data\n# data_iterable = iter(train_loader) # Converting our train_dataloader to iterable so that we can iterate through it.\n# images, labels = next(data_iterable) # Going from 1st batch of 100 images to the next batch\n# fig = plt.figure(figsize=(20,10))\n\n# # Let's plot 50 images from our train_dataset\n# for idx in np.arange(10):\n#     ax = fig.add_subplot(2, 5, idx+1, xticks=[], yticks=[])\n    \n#     # Note: imshow can't print tensor\n#     # Let's convert tensor image to numpy using im_convert function for imshow to print the image\n#     plt.imshow(im_convert(images[idx]))\n#     ax.set_title(classes[labels[idx].item()])","metadata":{"papermill":{"duration":0.019575,"end_time":"2024-08-05T09:56:10.770552","exception":false,"start_time":"2024-08-05T09:56:10.750977","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:52.848777Z","iopub.execute_input":"2024-08-05T15:11:52.849476Z","iopub.status.idle":"2024-08-05T15:11:52.860265Z","shell.execute_reply.started":"2024-08-05T15:11:52.849439Z","shell.execute_reply":"2024-08-05T15:11:52.859279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Classifier**","metadata":{"papermill":{"duration":0.011539,"end_time":"2024-08-05T09:56:10.793780","exception":false,"start_time":"2024-08-05T09:56:10.782241","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CosNorm_Classifier(nn.Module):\n    def __init__(self, in_dims, out_dims, scale=10.0):\n        super(CosNorm_Classifier, self).__init__()\n        self.scale = scale\n        self.weight = nn.Parameter(torch.Tensor(out_dims, in_dims))\n        nn.init.xavier_uniform_(self.weight) \n\n#     def reset_parameters(self):\n#         stdv = 1. / math.sqrt(self.weight.size(1))\n#         self.weight.data.uniform_(-stdv, stdv)\n\n    def forward(self, input):\n#         print(f\"Input shape to CosNorm_Classifier: {input.shape}\")  # Debug print\n        norm_x = torch.norm(input.clone(), 2, 1, keepdim=True)\n        ex = (norm_x / (1 + norm_x)) * (input / norm_x)\n        ew = self.weight / torch.norm(self.weight, 2, 1, keepdim=True)\n#         print(f\"Shape of ex: {ex.shape}, Shape of ew: {ew.shape}\")  # Debug print\n        return torch.mm(self.scale * ex, ew.t())","metadata":{"papermill":{"duration":0.021227,"end_time":"2024-08-05T09:56:10.826680","exception":false,"start_time":"2024-08-05T09:56:10.805453","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:11:52.861452Z","iopub.execute_input":"2024-08-05T15:11:52.861775Z","iopub.status.idle":"2024-08-05T15:11:52.874174Z","shell.execute_reply.started":"2024-08-05T15:11:52.861750Z","shell.execute_reply":"2024-08-05T15:11:52.873077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Necessary functions for Triplet Loss","metadata":{"papermill":{"duration":0.011466,"end_time":"2024-08-05T09:56:10.849809","exception":false,"start_time":"2024-08-05T09:56:10.838343","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class NPairsLoss(nn.Module):\n    def __init__(self, margin=1.0):\n        super(NPairsLoss, self).__init__()\n        self.margin = margin\n\n    def forward(self, embeddings, target):\n        loss = 0.0\n        batch_size = embeddings.size(0)\n        target = target.view(-1, 1)\n        \n        mask = torch.eq(target, target.T).float()\n        \n        for i in range(batch_size):\n            positive_mask = mask[i].unsqueeze(1).expand_as(embeddings)\n            negative_mask = (1 - mask[i]).unsqueeze(1).expand_as(embeddings)\n            anchor = embeddings[i]\n            positive_samples = embeddings * positive_mask\n            negative_samples = embeddings * negative_mask\n            \n            pos_distances = torch.sum((anchor - positive_samples) ** 2, dim=1)\n            neg_distances = torch.sum((anchor - negative_samples) ** 2, dim=1)\n            \n            loss += torch.sum(torch.relu(pos_distances - neg_distances + self.margin))\n        \n        loss /= batch_size\n        return loss","metadata":{"papermill":{"duration":0.033221,"end_time":"2024-08-05T09:56:10.894924","exception":false,"start_time":"2024-08-05T09:56:10.861703","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:29:25.065195Z","iopub.execute_input":"2024-08-05T15:29:25.066079Z","iopub.status.idle":"2024-08-05T15:29:25.075453Z","shell.execute_reply.started":"2024-08-05T15:29:25.066039Z","shell.execute_reply":"2024-08-05T15:29:25.074428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model**","metadata":{"papermill":{"duration":0.011544,"end_time":"2024-08-05T09:56:10.918161","exception":false,"start_time":"2024-08-05T09:56:10.906617","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class SimpleCNN(nn.Module):\n    def __init__(self):\n        super(SimpleCNN, self).__init__()\n        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n        self.fc1 = nn.Linear(64*8*8, 128)\n        self.fc2 = nn.Linear(128, 10) #CIFAR-10 has 10 classes\n        \n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = F.max_pool2d(x, 2)\n        x = F.relu(self.conv2(x))\n        x = F.max_pool2d(x, 2)\n        x = x.view(x.size(0), -1)\n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)\n        \n        return x","metadata":{"papermill":{"duration":0.021598,"end_time":"2024-08-05T09:56:10.951409","exception":false,"start_time":"2024-08-05T09:56:10.929811","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:29:25.653372Z","iopub.execute_input":"2024-08-05T15:29:25.653782Z","iopub.status.idle":"2024-08-05T15:29:25.662331Z","shell.execute_reply.started":"2024-08-05T15:29:25.653752Z","shell.execute_reply":"2024-08-05T15:29:25.661206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Base network creation function\nclass BaseNetwork(nn.Module):\n    def __init__(self, embedding_size):\n        super(BaseNetwork, self).__init__()\n        self.flatten = nn.Flatten()\n        self.fc1 = nn.Linear(3*32*32, 128)\n        self.dropout1 = nn.Dropout(0.1)\n        self.fc2 = nn.Linear(128, 128)\n        self.dropout2 = nn.Dropout(0.1)\n        self.fc3 = nn.Linear(128, embedding_size)\n        \n    def forward(self, x):\n        x = self.flatten(x)\n        x = F.relu(self.fc1(x))\n        x = self.dropout1(x)\n        x = F.relu(self.fc2(x))\n        x = self.dropout2(x)\n        x = self.fc3(x)\n        return x","metadata":{"papermill":{"duration":0.021169,"end_time":"2024-08-05T09:56:10.984170","exception":false,"start_time":"2024-08-05T09:56:10.963001","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:29:25.911323Z","iopub.execute_input":"2024-08-05T15:29:25.912264Z","iopub.status.idle":"2024-08-05T15:29:25.919941Z","shell.execute_reply.started":"2024-08-05T15:29:25.912206Z","shell.execute_reply":"2024-08-05T15:29:25.918871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize model, Optimizer\nembedded_size = 128\n# model = SimpleCNN()\nbase_network = BaseNetwork(embedded_size)\n# Initialize the NPairs loss function\nnpairs_loss_fn = NPairsLoss(margin=1.0)\noptimizer = optim.Adam(base_network.parameters(), lr=0.0001)","metadata":{"papermill":{"duration":0.045091,"end_time":"2024-08-05T09:56:11.040787","exception":false,"start_time":"2024-08-05T09:56:10.995696","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:29:26.603345Z","iopub.execute_input":"2024-08-05T15:29:26.604407Z","iopub.status.idle":"2024-08-05T15:29:26.615103Z","shell.execute_reply.started":"2024-08-05T15:29:26.604369Z","shell.execute_reply":"2024-08-05T15:29:26.614284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get embeddings before training\nbase_network.eval()\nembeddings_before_train = []\nlabels_before_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        embeddings = base_network(data)\n        embeddings_before_train.append(embeddings.cpu().numpy())\n        labels_before_train.append(target.cpu().numpy())\n        \nembeddings_before_train = np.concatenate(embeddings_before_train)\nlabels_before_train = np.concatenate(labels_before_train)","metadata":{"papermill":{"duration":7.91435,"end_time":"2024-08-05T09:56:18.966766","exception":false,"start_time":"2024-08-05T09:56:11.052416","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:29:26.968244Z","iopub.execute_input":"2024-08-05T15:29:26.968627Z","iopub.status.idle":"2024-08-05T15:29:35.260587Z","shell.execute_reply.started":"2024-08-05T15:29:26.968600Z","shell.execute_reply":"2024-08-05T15:29:35.259114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Loop\nnum_epochs = 10\ntraining_loss = []\nbase_network.train()\nfor epoch in range(num_epochs):\n    epoch_loss = 0\n    for data, target, index in train_loader:\n        optimizer.zero_grad()\n        embeddings = base_network(data)\n        loss = npairs_loss_fn(embeddings, target)\n        loss.backward()\n        optimizer.step()\n        epoch_loss += loss.item()\n    training_loss.append(epoch_loss / len(train_loader))\n        \n    print(f'Epoch {epoch + 1}/{num_epochs}, Loss: {epoch_loss / len(train_loader): .4f}')\n    \n    # Evaluate in validation set\n    base_network.eval()\n    all_targets = []\n    all_embeddings = []\n    all_predictions = []\n    start_time = time.time()\n    with torch.no_grad():\n        for data, target, index in validation_loader:\n            embeddings = base_network(data)\n            outputs = base_network(data)\n            predictions = torch.argmax(outputs, dim=1)\n            all_targets.extend(target.cpu().numpy())\n            all_embeddings.extend(embeddings.cpu().numpy())\n            all_predictions.extend(predictions.cpu().numpy())\n    end_time = time.time()\n    testing_time = end_time - start_time\n    \n    # Calculate metrics\n    accuracy = accuracy_score(all_targets, all_predictions)\n    balanced_acc = balanced_accuracy_score(all_targets, all_predictions)\n    f1 = f1_score(all_targets, all_predictions, average='weighted')\n    recall = recall_score(all_targets, all_predictions, average='weighted')\n    precision = precision_score(all_targets, all_predictions, average='weighted')\n    \n    print(f'Epoch {epoch+1}, Accuracy: {accuracy:.4f}, Balanced Accuracy: {balanced_acc:.4f},'\n          f'F1 Score: {f1: .4f}, Recall: {recall: .4f}, Precision: {precision: .4f},'\n          f'Testing Time: {testing_time: .4f}s')","metadata":{"papermill":{"duration":348.387588,"end_time":"2024-08-05T10:02:07.366727","exception":false,"start_time":"2024-08-05T09:56:18.979139","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:29:35.262813Z","iopub.execute_input":"2024-08-05T15:29:35.263237Z","iopub.status.idle":"2024-08-05T15:37:16.535634Z","shell.execute_reply.started":"2024-08-05T15:29:35.263185Z","shell.execute_reply":"2024-08-05T15:37:16.534400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After training, plot ROC Curve\nbase_network.eval()\nwith torch.no_grad():\n    # Convert lists to numpy arrays\n    all_targets = np.array(all_targets)\n    all_embeddings = np.array(all_embeddings)\n    \n    # Binarize the labels for multi-class ROC computation\n    y_bin = label_binarize(all_targets, classes=np.arange(10))  # Adjust `classes` based on the number of classes\n    y_score = np.array(all_embeddings)  # Adjust this if you need to obtain scores from embeddings\n    \n    # Plot ROC curves for each class\n    plt.figure(figsize=(10, 8))\n    for i in range(10):  # Assuming 10 classes\n        fpr, tpr, _ = roc_curve(y_bin[:, i], y_score[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label=f'Class {i} (area = {roc_auc:.2f})')\n    \n    plt.plot([0, 1], [0, 1], color='gray', linestyle='--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic for Multi-Class')\n    plt.legend(loc='lower right')\n    plt.show()","metadata":{"papermill":{"duration":0.649278,"end_time":"2024-08-05T10:02:08.030058","exception":false,"start_time":"2024-08-05T10:02:07.380780","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:39:52.578015Z","iopub.execute_input":"2024-08-05T15:39:52.578485Z","iopub.status.idle":"2024-08-05T15:39:53.172185Z","shell.execute_reply.started":"2024-08-05T15:39:52.578457Z","shell.execute_reply":"2024-08-05T15:39:53.171198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get embeddings after train\nbase_network.eval()\nembeddings_after_train = []\nlabels_after_train = []\nwith torch.no_grad():\n    for data, target, index in validation_loader:\n        embeddings = base_network(data)\n        embeddings_after_train.append(embeddings.cpu().numpy())\n        labels_after_train.append(target.cpu().numpy())\n\nembeddings_after_train = np.concatenate(embeddings_after_train)\nlabels_after_train = np.concatenate(labels_after_train)","metadata":{"papermill":{"duration":7.805645,"end_time":"2024-08-05T10:02:15.851875","exception":false,"start_time":"2024-08-05T10:02:08.046230","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:39:58.460953Z","iopub.execute_input":"2024-08-05T15:39:58.461344Z","iopub.status.idle":"2024-08-05T15:40:07.065667Z","shell.execute_reply.started":"2024-08-05T15:39:58.461313Z","shell.execute_reply":"2024-08-05T15:40:07.064294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the effect of embeddings using PCA\nnum_components=2\nstep = 5\n\npca = PCA(n_components=num_components)\ndecomposed_before_train = pca.fit_transform(embeddings_before_train)\ndecomposed_after_train = pca.fit_transform(embeddings_after_train)\ntest_class_labels = np.unique(labels_after_train)\n\nfig = plt.figure(figsize=(16, 8))\nfor label in test_class_labels:\n    decomposed_before_class = decomposed_before_train[labels_before_train == label]\n    decomposed_after_class = decomposed_after_train[labels_after_train == label]\n    \n    plt.subplot(1, 2, 1)\n    plt.scatter(decomposed_before_class[:, 0], decomposed_before_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('Before Training (embeddings)')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.scatter(decomposed_after_class[:, 0], decomposed_after_class[:, 1], label=str(label), alpha=0.5)\n    plt.title('After {num_epochs} Epochs (embeddings)')\n    plt.legend()\n    \nplt.show()","metadata":{"papermill":{"duration":2.7778,"end_time":"2024-08-05T10:02:18.646491","exception":false,"start_time":"2024-08-05T10:02:15.868691","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:40:26.121159Z","iopub.execute_input":"2024-08-05T15:40:26.122099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the trained model\ntorch.save(base_network.state_dict(), 'cifar10_base_network.pth')","metadata":{"papermill":{"duration":0.03584,"end_time":"2024-08-05T10:02:18.706684","exception":false,"start_time":"2024-08-05T10:02:18.670844","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:40:34.949053Z","iopub.execute_input":"2024-08-05T15:40:34.949765Z","iopub.status.idle":"2024-08-05T15:40:34.959715Z","shell.execute_reply.started":"2024-08-05T15:40:34.949734Z","shell.execute_reply":"2024-08-05T15:40:34.958372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the training data\nplt.figure(figsize=(8,8))\nplt.plot(training_loss, label='Training Loss')\nplt.legend()\nplt.title('Training Loss')\nplt.show()","metadata":{"papermill":{"duration":0.261296,"end_time":"2024-08-05T10:02:18.991901","exception":false,"start_time":"2024-08-05T10:02:18.730605","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-08-05T15:40:36.231815Z","iopub.execute_input":"2024-08-05T15:40:36.232583Z","iopub.status.idle":"2024-08-05T15:40:36.474798Z","shell.execute_reply.started":"2024-08-05T15:40:36.232549Z","shell.execute_reply":"2024-08-05T15:40:36.472774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.02436,"end_time":"2024-08-05T10:02:19.041740","exception":false,"start_time":"2024-08-05T10:02:19.017380","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}