{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Metric Learning on MNIST using PyTorch.\n\nSTEPS:\n- Train a fully connected using MNIST\n\n- The architecture of NN: 786, 256, 2, 10\n\n- Why 2 neurons at the pre-final layer?\n    - Because we want to show embeddings of each image in 2D..\n\n- Visualize embedding\n    - Mostly elongated clusters.\n\n- Same exercise for CNN with 2 neurons at pre-final layer\n\n- Similar visualization\n\n- Train Siamese network\n    - Dataset class implementation\n    - Custom Loss function\n    - Difference between train-time and test-time\n\n- Visualize embeddings\n    - Clusters with nice circular shape can be visualized.\n\n- Repeat training and visualization for Triplet network\n    - Training is faster. Clusters are better than siamese.\n    \n- Use trained NN on Jetson Nano for inference","metadata":{"execution":{"iopub.status.busy":"2021-10-08T08:58:57.891551Z","iopub.execute_input":"2021-10-08T08:58:57.891811Z","iopub.status.idle":"2021-10-08T08:58:57.920684Z","shell.execute_reply.started":"2021-10-08T08:58:57.891783Z","shell.execute_reply":"2021-10-08T08:58:57.919717Z"}}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport numpy as np\nfrom torch.utils.data import Dataset, DataLoader\nimport random\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:15.767738Z","iopub.execute_input":"2021-10-09T02:44:15.769199Z","iopub.status.idle":"2021-10-09T02:44:16.993931Z","shell.execute_reply.started":"2021-10-09T02:44:15.769138Z","shell.execute_reply":"2021-10-09T02:44:16.992787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Few hyper-parametes","metadata":{}},{"cell_type":"code","source":"num_epochs = 50\nbatch_size = 4096\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:21.403935Z","iopub.execute_input":"2021-10-09T02:44:21.404398Z","iopub.status.idle":"2021-10-09T02:44:21.410604Z","shell.execute_reply.started":"2021-10-09T02:44:21.404357Z","shell.execute_reply":"2021-10-09T02:44:21.409435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MNIST dataset provided in pytorch for training and validation","metadata":{"execution":{"iopub.status.busy":"2021-10-08T10:26:24.898392Z","iopub.execute_input":"2021-10-08T10:26:24.898732Z","iopub.status.idle":"2021-10-08T10:26:24.905386Z","shell.execute_reply.started":"2021-10-08T10:26:24.898701Z","shell.execute_reply":"2021-10-08T10:26:24.903793Z"}}},{"cell_type":"code","source":"transforms_ = transforms.ToTensor()\ntrain_data = datasets.MNIST(root='./data', train=True, transform=transforms_, download=True)\ntest_data = datasets.MNIST(root='./data', train=False, transform=transforms_, download=True)\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:22.001387Z","iopub.execute_input":"2021-10-09T02:44:22.001817Z","iopub.status.idle":"2021-10-09T02:44:24.693009Z","shell.execute_reply.started":"2021-10-09T02:44:22.001786Z","shell.execute_reply":"2021-10-09T02:44:24.692064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fully connected NN","metadata":{}},{"cell_type":"code","source":"class FullyConnectedNet(nn.Module):\n    def __init__(self):\n        super(FullyConnectedNet,self).__init__()\n        self.fc1 = nn.Linear(784, 256)\n        self.relu = nn.ReLU()\n        self.fc2 = nn.Linear(256, 2)\n        self.relu = nn.ReLU()\n        self.fc4 = nn.Linear(2, 10)\n        self.bn1 = nn.BatchNorm1d(256)\n        self.bn3 = nn.BatchNorm1d(2)\n\n    def forward(self,x):\n        out = self.fc1(x)\n        out = self.relu(out)\n        out = self.bn1(out)\n        out = self.fc2(out)\n        out = self.relu(out)\n        out = self.bn3(out)\n        out = self.fc4(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:24.775184Z","iopub.execute_input":"2021-10-09T02:44:24.776436Z","iopub.status.idle":"2021-10-09T02:44:24.787163Z","shell.execute_reply.started":"2021-10-09T02:44:24.77637Z","shell.execute_reply":"2021-10-09T02:44:24.786391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Fully connected NN","metadata":{}},{"cell_type":"code","source":"classifier_fc = FullyConnectedNet().to(device)\nloss_function = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(classifier_fc.parameters(), lr=1e-1)\n\nfor epoch in range(num_epochs):\n    running_loss_train = 0.\n    running_corrects_train = 0\n    classifier_fc.train()\n    for i ,(images,labels) in enumerate(train_loader):    \n        images = images.view(-1,28*28).to(device)\n        labels = labels.to(device)\n        optimizer.zero_grad()\n        outputs = classifier_fc(images)\n        loss = loss_function(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        _, preds = torch.max(outputs, 1)\n        running_loss_train += loss.item() * images.size(0)\n        running_corrects_train += torch.sum(preds == labels.data)\n    epoch_loss_train = running_loss_train / len(train_data)\n    epoch_acc_train = running_corrects_train.double() / len(train_data)\n\n    classifier_fc.eval()\n    running_loss_test = 0.\n    running_corrects_test = 0\n    for i ,(images,labels) in enumerate(test_loader):    \n        images = images.view(-1,28*28).to(device)\n        labels = labels.to(device)\n        outputs = classifier_fc(images)\n        loss = loss_function(outputs, labels)\n        _, preds = torch.max(outputs, 1)\n        running_loss_test += loss.item() * images.size(0)\n        running_corrects_test += torch.sum(preds == labels.data)\n    epoch_loss_test = running_loss_test / len(test_data)\n    epoch_acc_test = running_corrects_test.double() / len(test_data)         \n\n    print(f'Epoch {epoch+1}/{num_epochs}\\t'\n          f'Train loss: {epoch_loss_train:.4f} | '\n          f'Train accuracy: {epoch_acc_train:.4f} | '\n          f'Test loss: {epoch_loss_test:.4f} | '\n          f'Test accuracy: {epoch_acc_test:.4f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are going to visualize the features extracted by fully connected NN in pre-final layer. Our initial implentation does not return features in forward method, so we are writing new class for fully connected NN. We will copy weights of trained NN in newly implemented NN class.","metadata":{}},{"cell_type":"code","source":"class FullyConnectedNet_features(nn.Module):\n    def __init__(self):\n        super(FullyConnectedNet_features,self).__init__()\n        self.fc1 = nn.Linear(784, 256)\n        self.relu = nn.ReLU()\n        self.fc2 = nn.Linear(256, 2)\n        self.fc4 = nn.Linear(2, 10)\n        self.bn1 = nn.BatchNorm1d(256)\n        self.bn3 = nn.BatchNorm1d(2)\n\n    def forward(self,x):\n        out = self.fc1(x)\n        out = self.relu(out)\n        out = self.bn1(out)\n        out = self.fc2(out)\n        features = out\n        out = self.relu(out)\n        out = self.bn3(out)\n        out = self.fc4(out)\n        return out, features","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:31.424651Z","iopub.execute_input":"2021-10-09T02:44:31.424968Z","iopub.status.idle":"2021-10-09T02:44:31.434725Z","shell.execute_reply.started":"2021-10-09T02:44:31.424935Z","shell.execute_reply":"2021-10-09T02:44:31.433653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Colors for vizualization","metadata":{}},{"cell_type":"code","source":"clr = [\n    '#1f77b4',  # muted blue\n    '#ff7f0e',  # safety orange\n    '#2ca02c',  # cooked asparagus green\n    '#d62728',  # brick red\n    '#9467bd',  # muted purple\n    '#8c564b',  # chestnut brown\n    '#e377c2',  # raspberry yogurt pink\n    '#7f7f7f',  # middle gray\n    '#bcbd22',  # curry yellow-green\n    '#17becf'   # blue-teal\n]","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:32.768262Z","iopub.execute_input":"2021-10-09T02:44:32.769319Z","iopub.status.idle":"2021-10-09T02:44:32.77456Z","shell.execute_reply.started":"2021-10-09T02:44:32.76926Z","shell.execute_reply":"2021-10-09T02:44:32.773549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot 2D features learnt by fully connected NN","metadata":{}},{"cell_type":"code","source":"classifier_with_features = FullyConnectedNet_features().to(device)\nclassifier_with_features.load_state_dict(classifier_fc.state_dict())\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=len(train_data))\ntest_loader = torch.utils.data.DataLoader(dataset=test_data, batch_size=len(test_data))\n\nclassifier_with_features.eval()\n\nwith torch.no_grad():\n    images, train_labels = next(iter(train_loader))\n    _, train_features = classifier_with_features(images.view(-1,28*28).to(device))\n    train_features = train_features.cpu()\n\n    images, test_labels = next(iter(test_loader))\n    _, test_features = classifier_with_features(images.view(-1,28*28).to(device))\n    test_features = test_features.cpu()\n\nfigure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')\n\nplt.subplot(121)\nfor i in range(10): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(122)\nfor i in range(10): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i],s=1, label=f'{i}')\nplt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Similar excercise for CNN","metadata":{}},{"cell_type":"code","source":"class CNNClassifier(nn.Module):\n    def __init__(self):\n        super(CNNClassifier,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 16, 5)\n        self.fc1 = nn.Linear(16*4*4, 32)\n        self.fc2 = nn.Linear(32, 2)\n        self.relu = nn.ReLU()\n        self.fc3 = nn.Linear(2, 10)\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(16)\n        self.bn4 = nn.BatchNorm1d(2)\n        self.bn3 = nn.BatchNorm1d(32)\n\n    def forward(self,x):\n        out = self.conv1(x)\n        out = self.maxpool(out)\n        out = self.relu(out)\n        out = self.bn1(out)\n        out = self.conv2(out)\n        out = self.maxpool(out)\n        out = self.relu(out)\n        out = self.bn2(out)\n        out = out.view(out.size()[0], -1)\n        out = self.fc1(out)\n        out = self.relu(out)\n        out = self.bn3(out)\n        out = self.fc2(out)\n        features = out\n        out = self.relu(out)\n        out = self.bn4(out)        \n        out = self.fc3(out) \n        return out, features","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:36.693954Z","iopub.execute_input":"2021-10-09T02:44:36.694307Z","iopub.status.idle":"2021-10-09T02:44:36.707089Z","shell.execute_reply.started":"2021-10-09T02:44:36.694275Z","shell.execute_reply":"2021-10-09T02:44:36.706374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = transforms.Compose([\n    transforms.Resize(size=(28, 28)), \n    transforms.ToTensor(), \n    transforms.Normalize((0.1307,), (0.3081,))\n])\n\ntrain_data = datasets.MNIST(root='./data', train=True, transform=data_transforms, download=True)\ntest_data = datasets.MNIST(root='./data', train=False, transform=data_transforms, download=True)\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:37.464643Z","iopub.execute_input":"2021-10-09T02:44:37.465185Z","iopub.status.idle":"2021-10-09T02:44:37.511681Z","shell.execute_reply.started":"2021-10-09T02:44:37.465145Z","shell.execute_reply":"2021-10-09T02:44:37.510208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_classifier = CNNClassifier().to(device)\nloss_function = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(cnn_classifier.parameters(), lr=0.1)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:38.971Z","iopub.execute_input":"2021-10-09T02:44:38.972153Z","iopub.status.idle":"2021-10-09T02:44:39.001393Z","shell.execute_reply.started":"2021-10-09T02:44:38.972081Z","shell.execute_reply":"2021-10-09T02:44:39.000356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    running_loss_train = 0.\n    running_corrects_train = 0\n    cnn_classifier.train()\n    for i ,(images,labels) in enumerate(train_loader):    \n        images = images.to(device)\n        labels = labels.to(device)\n        optimizer.zero_grad()\n        outputs, features = cnn_classifier(images)\n        loss = loss_function(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        _, preds = torch.max(outputs, 1)\n        running_loss_train += loss.item() * images.size(0)\n        running_corrects_train += torch.sum(preds == labels.data)\n    epoch_loss_train = running_loss_train / len(train_data)\n    epoch_acc_train = running_corrects_train.double() / len(train_data)\n\n    cnn_classifier.eval()\n    running_loss_test = 0.\n    running_corrects_test = 0\n    for i ,(images,labels) in enumerate(test_loader):    \n        images = images.to(device)\n        labels = labels.to(device)\n        outputs, features = cnn_classifier(images)\n        loss = loss_function(outputs, labels)\n        _, preds = torch.max(outputs, 1)\n        running_loss_test += loss.item() * images.size(0)\n        running_corrects_test += torch.sum(preds == labels.data)\n    epoch_loss_test = running_loss_test / len(test_data)\n    epoch_acc_test = running_corrects_test.double() / len(test_data)         \n\n    print(f'Epoch {epoch+1}/{num_epochs}\\t'\n          f'Train loss: {epoch_loss_train:.4f} | '\n          f'Train accuracy: {epoch_acc_train:.4f} | '\n          f'Test loss: {epoch_loss_test:.4f} | '\n          f'Test accuracy: {epoch_acc_test:.4f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset=train_data, batch_size=len(train_data))\ntest_loader = DataLoader(dataset=test_data, batch_size=len(test_data))\ncnn_classifier.eval()\n\nwith torch.no_grad():\n    images, train_labels = next(iter(train_loader))\n    _, train_features = cnn_classifier(images.to(device))\n    train_features = train_features.cpu()\n    \n    images, test_labels = next(iter(test_loader))\n    _, test_features = cnn_classifier(images.to(device))\n    test_features = test_features.cpu()\n\n\nfigure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.subplot(121)\nfor i in range(10): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(122)\nfor i in range(10): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i], s=1, label=f'{i}')\nplt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Siamese network -Training\n- While training, we pass two images through CNN and calculate loss using embeddings of pair of image.\n- forward method takes two images as input, passes each image through the same CNN and returns embeddings of each image","metadata":{}},{"cell_type":"code","source":"class SiameseNet(nn.Module):\n    def __init__(self):\n        super(SiameseNet,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 2)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self,x1, x2):\n        out1 = self.conv1(x1)\n        out1 = self.maxpool(out1)\n        out1 = self.relu(out1)\n        out1 = self.bn1(out1)\n        out1 = self.conv2(out1)\n        out1 = self.maxpool(out1)\n        out1 = self.relu(out1)\n        out1 = self.bn2(out1)\n        out1 = out1.view(out1.size()[0], -1)\n        out1 = self.fc1(out1)\n        \n        out2 = self.conv1(x2)\n        out2 = self.maxpool(out2)\n        out2 = self.relu(out2)\n        out2 = self.bn1(out2)\n        out2 = self.conv2(out2)\n        out2 = self.maxpool(out2)\n        out2 = self.relu(out2)\n        out2 = self.bn2(out2)\n        out2 = out2.view(out2.size()[0], -1)\n        out2 = self.fc1(out2)\n\n        return out1, out2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset class\n- returns pair of images with label indicating if pair is positive(simialr) or negative(disimilar)","metadata":{}},{"cell_type":"code","source":"class siameseMNIST(Dataset):\n    def __init__(self, phase):\n        self.samples = []\n        self.phase = phase\n        self.train_indices = []\n        self.test_indices = []\n        for i in range(10):\n            self.train_indices.append([x for x, item in enumerate(train_data.targets == i) if item])\n            self.test_indices.append([x for x, item in enumerate(test_data.targets == i) if item])\n\n    def __getitem__(self, index):\n        if self.phase == 'train':\n            x1, label = train_data.__getitem__(index)\n            if random.random() > 0.5:\n                x2, _ = train_data.__getitem__(random.choice(self.train_indices[label]))\n                pair_label = 1\n            else:\n                all_classes = list(range(10))\n                _ = all_classes.pop(label)\n                diff_class = random.choice(all_classes)\n                x2, _ = train_data.__getitem__(random.choice(self.train_indices[diff_class]))\n                pair_label = 0\n            return x1, x2, pair_label \n\n        else:\n            x1, label = test_data.__getitem__(index)\n            if random.random() > 0.5:\n                x2, _ = test_data.__getitem__(random.choice(self.test_indices[label]))\n                pair_label = 1\n            else:\n                all_classes = list(range(10))\n                _ = all_classes.pop(label)\n                diff_class = random.choice(all_classes)\n                x2, _ = test_data.__getitem__(random.choice(self.test_indices[diff_class]))\n                pair_label = 0\n            return x1, x2, pair_label\n\n    def __len__(self):\n        if self.phase == 'train': return 60000\n        return 10000","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"siamese_train_data = siameseMNIST(phase='train')\nsiamese_test_data = siameseMNIST(phase='test')\n\ntrain_loader = DataLoader(dataset=siamese_train_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=siamese_test_data, batch_size=batch_size, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Contrastive Loss","metadata":{}},{"cell_type":"code","source":"class ContrastiveLoss(nn.Module):\n    def __init__(self, margin):\n        super(ContrastiveLoss, self).__init__()\n        self.margin = margin\n        self.eps = 1e-9\n\n    def forward(self, output1, output2, target):\n        distances = (output2 - output1).pow(2).sum(1)\n        losses = 0.5 * (target.float() * distances +\n                        (1 + -1 * target).float() * F.relu(self.margin - (distances + self.eps).sqrt()).pow(2))\n        return losses.mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"siamese_net = SiameseNet().to(device)\nloss_function = ContrastiveLoss(1.)\noptimizer = torch.optim.Adam(siamese_net.parameters(), lr=1e-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    siamese_net.train()\n    running_loss_train = 0.\n    for i ,(images_1, images_2,labels) in enumerate(train_loader):    \n        images_1 = images_1.to(device)\n        images_2 = images_2.to(device)\n        labels = labels.to(device)\n        optimizer.zero_grad()\n        output_1, output_2 = siamese_net(images_1, images_2)\n        loss = loss_function(output_1, output_2, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss_train += loss.item() * images_1.size(0)\n    epoch_loss_train = running_loss_train / len(siamese_train_data)\n\n    siamese_net.eval()\n    running_loss_test = 0.\n    for i ,(images_1, images_2,labels) in enumerate(test_loader):    \n        images_1 = images_1.to(device)\n        images_2 = images_2.to(device)\n        labels = labels.to(device)\n        output_1, output_2 = siamese_net(images_1, images_2)\n        loss = loss_function(output_1, output_2, labels)\n        running_loss_test += loss.item() * images_1.size(0)\n    epoch_loss_test = running_loss_test / len(siamese_test_data)\n\n    print(f'Epoch {epoch+1}/{num_epochs} | '\n          f'Train loss: {epoch_loss_train} | '\n          f'Test loss: {epoch_loss_test}')\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class for test-time siamese","metadata":{}},{"cell_type":"code","source":"class siamese_test(nn.Module):\n    def __init__(self):\n        super(siamese_test,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 2)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self,x1):\n        out1 = self.conv1(x1)\n        out1 = self.maxpool(out1)\n        out1 = self.relu(out1)\n        out1 = self.bn1(out1)\n        out1 = self.conv2(out1)\n        out1 = self.maxpool(out1)\n        out1 = self.relu(out1)\n        out1 = self.bn2(out1)\n        out1 = out1.view(out1.size()[0], -1)\n        out1 = self.fc1(out1)\n        return out1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"siamese_test_net = siamese_test().to(device)\nsiamese_test_net.load_state_dict(siamese_net.state_dict())\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=len(train_data))\ntest_loader = DataLoader(dataset=test_data, batch_size=len(test_data))\n\nsiamese_test_net.eval()\n\nwith torch.no_grad():\n    images, train_labels = next(iter(train_loader))\n    train_features = siamese_test_net(images.to(device))\n    train_features = train_features.cpu()\n    \n    images, test_labels = next(iter(test_loader))\n    test_features = siamese_test_net(images.to(device))\n    test_features = test_features.cpu()\n\nfigure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')\n\nplt.subplot(121)\nfor i in range(10): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(122)\nfor i in range(10): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i], s=1, label=f'{i}')\nplt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Triplet loss","metadata":{}},{"cell_type":"code","source":"class TripletLoss(nn.Module):\n    def __init__(self, margin):\n        super(TripletLoss, self).__init__()\n        self.margin = margin\n\n    def forward(self, anchor, positive, negative, size_average=True):\n        distance_positive = (anchor - positive).pow(2).sum(1)  # .pow(.5)\n        distance_negative = (anchor - negative).pow(2).sum(1)  # .pow(.5)\n        losses = F.relu(distance_positive - distance_negative + self.margin)\n        return losses.mean() if size_average else losses.sum()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:55.512781Z","iopub.execute_input":"2021-10-09T02:44:55.513585Z","iopub.status.idle":"2021-10-09T02:44:55.521331Z","shell.execute_reply.started":"2021-10-09T02:44:55.513542Z","shell.execute_reply":"2021-10-09T02:44:55.52024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class tripletMNIST(Dataset):\n    def __init__(self, phase):\n        self.samples = []\n        self.phase = phase\n        self.train_indices = []\n        self.test_indices = []\n        for i in range(10):\n            self.train_indices.append([x for x, item in enumerate(train_data.targets == i) if item])\n            self.test_indices.append([x for x, item in enumerate(test_data.targets == i) if item])\n\n    def __getitem__(self, index):\n        if self.phase == 'train':\n            anchor, label = train_data.__getitem__(index) ## __getitem__ of MNIST dataset class\n            pos, _ = train_data.__getitem__(random.choice(self.train_indices[label]))\n            all_classes = list(range(10))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = train_data.__getitem__(random.choice(self.train_indices[diff_class]))\n            return anchor, pos, neg\n\n        if self.phase == 'test':\n            anchor, label = test_data.__getitem__(index)\n            pos, _ = test_data.__getitem__(random.choice(self.test_indices[label]))\n            all_classes = list(range(10))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = test_data.__getitem__(random.choice(self.test_indices[diff_class]))\n            return anchor, pos, neg\n            \n    def __len__(self):\n        if self.phase == 'train': return 60000\n        return 10000","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:44:56.84767Z","iopub.execute_input":"2021-10-09T02:44:56.848815Z","iopub.status.idle":"2021-10-09T02:44:56.86159Z","shell.execute_reply.started":"2021-10-09T02:44:56.848773Z","shell.execute_reply":"2021-10-09T02:44:56.860661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_train_data = tripletMNIST(phase='train')\ntriplet_test_data = tripletMNIST(phase='test')\n\ntrain_loader = DataLoader(dataset=triplet_train_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=triplet_test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:45:04.659639Z","iopub.execute_input":"2021-10-09T02:45:04.660978Z","iopub.status.idle":"2021-10-09T02:45:10.502515Z","shell.execute_reply.started":"2021-10-09T02:45:04.660899Z","shell.execute_reply":"2021-10-09T02:45:10.501657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletNet(nn.Module):\n    def __init__(self):\n        super(TripletNet,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 2)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self, anchor, pos, neg):\n        anchor_embd = self.conv1(anchor)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn1(anchor_embd)\n        anchor_embd = self.conv2(anchor_embd)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn2(anchor_embd)\n        anchor_embd = anchor_embd.view(anchor_embd.size()[0], -1)\n        anchor_embd = self.fc1(anchor_embd)\n\n        pos_embd = self.conv1(pos)\n        pos_embd = self.maxpool(pos_embd)\n        pos_embd = self.relu(pos_embd)\n        pos_embd = self.bn1(pos_embd)\n        pos_embd = self.conv2(pos_embd)\n        pos_embd = self.maxpool(pos_embd)\n        pos_embd = self.relu(pos_embd)\n        pos_embd = self.bn2(pos_embd)\n        pos_embd = pos_embd.view(pos_embd.size()[0], -1)\n        pos_embd = self.fc1(pos_embd)\n\n        neg_embd = self.conv1(neg)\n        neg_embd = self.maxpool(neg_embd)\n        neg_embd = self.relu(neg_embd)\n        neg_embd = self.bn1(neg_embd)\n        neg_embd = self.conv2(neg_embd)\n        neg_embd = self.maxpool(neg_embd)\n        neg_embd = self.relu(neg_embd)\n        neg_embd = self.bn2(neg_embd)\n        neg_embd = neg_embd.view(neg_embd.size()[0], -1)\n        neg_embd = self.fc1(neg_embd)\n\n        return anchor_embd, pos_embd, neg_embd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_net = TripletNet().to(device)\nloss_function = TripletLoss(1.)\noptimizer = torch.optim.Adam(triplet_net.parameters(), lr=1e-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    triplet_net.train()\n    running_loss_train = 0.\n    for i ,(anchor, pos, neg) in enumerate(train_loader):  \n        anchor = anchor.to(device)\n        pos = pos.to(device)\n        neg = neg.to(device)\n        optimizer.zero_grad()\n        anchor_embd, pos_embd, neg_embd = triplet_net(anchor, pos, neg)\n        loss = loss_function(anchor_embd, pos_embd, neg_embd)\n        loss.backward()\n        optimizer.step()\n        running_loss_train += loss.item() * anchor.size(0)\n    epoch_loss_train = running_loss_train / len(triplet_train_data)\n\n    triplet_net.eval()\n    running_loss_test = 0.\n    for i ,(anchor, pos, neg) in enumerate(test_loader):  \n        anchor = anchor.to(device)\n        pos = pos.to(device)\n        neg = neg.to(device)\n        anchor_embd, pos_embd, neg_embd = triplet_net(anchor, pos, neg)\n        loss = loss_function(anchor_embd, pos_embd, neg_embd)\n        running_loss_test += loss.item() * anchor.size(0)\n    epoch_loss_test = running_loss_test / len(triplet_test_data)\n\n    print(f'Epoch {epoch+1}/{num_epochs} | '\n        f'Train loss: {epoch_loss_train} | '\n        f'Test loss: {epoch_loss_test}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletNetTest(nn.Module):\n    def __init__(self):\n        super(TripletNetTest,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 2)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self, anchor):\n        anchor_embd = self.conv1(anchor)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn1(anchor_embd)\n        anchor_embd = self.conv2(anchor_embd)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn2(anchor_embd)\n        anchor_embd = anchor_embd.view(anchor_embd.size()[0], -1)\n        anchor_embd = self.fc1(anchor_embd)\n        return anchor_embd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_test_net = TripletNetTest().to(device)\ntriplet_test_net.load_state_dict(triplet_net.state_dict())\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=len(train_data))\ntest_loader = torch.utils.data.DataLoader(dataset=test_data, batch_size=len(test_data))\ntriplet_test_net.eval()\n\nwith torch.no_grad():\n    images, train_labels = next(iter(train_loader))\n    train_features = triplet_test_net(images.to(device))\n    train_features = train_features.cpu()\n    \n    images, test_labels = next(iter(test_loader))\n    test_features = triplet_test_net(images.to(device))\n    test_features = test_features.cpu()\n\n\nfigure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.subplot(121)\nfor i in range(10): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(122)\nfor i in range(10): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i], s=1, label=f'{i}')\nplt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train on five classes, test on all","metadata":{}},{"cell_type":"code","source":"class tripletMNIST(Dataset):\n    def __init__(self, phase):\n        self.samples = []\n        self.phase = phase\n        self.train_indices = []\n        self.test_indices = []\n        for i in range(10):\n            self.train_indices.append([x for x, item in enumerate(train_data.targets == i) if item])\n            self.test_indices.append([x for x, item in enumerate(test_data.targets == i) if item])\n\n    def __getitem__(self, index):\n        if self.phase == 'train':\n            anchor, label = train_data.__getitem__(index) ## __getitem__ of MNIST dataset class\n            pos, _ = train_data.__getitem__(random.choice(self.train_indices[label]))\n            all_classes = list(range(10))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = train_data.__getitem__(random.choice(self.train_indices[diff_class]))\n            return anchor, pos, neg\n\n        if self.phase == 'test':\n            anchor, label = test_data.__getitem__(index)\n            pos, _ = test_data.__getitem__(random.choice(self.test_indices[label]))\n            all_classes = list(range(10))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = test_data.__getitem__(random.choice(self.test_indices[diff_class]))\n            return anchor, pos, neg\n            \n    def __len__(self):\n        if self.phase == 'train': return 60000\n        return 10000\n\n    \n\nclass tripletMNIST5(Dataset):\n    def __init__(self, phase):\n        self.samples = []\n        self.phase = phase\n        self.train_indices = []\n        self.test_indices = []\n        for i in range(5):\n            self.train_indices.append([x for x, item in enumerate(train_data.targets == i) if item])\n            self.test_indices.append([x for x, item in enumerate(test_data.targets == i) if item])\n\n    def __getitem__(self, index):\n        if self.phase == 'train':\n            label = 9\n            labels_ = [5,6,7,8,9]\n            while label in labels_:\n                anchor, label = train_data.__getitem__(index) ## __getitem__ of MNIST dataset class\n                index = random.randint(0, 60000-1)\n            pos, _ = train_data.__getitem__(random.choice(self.train_indices[label]))\n            all_classes = list(range(5))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = train_data.__getitem__(random.choice(self.train_indices[diff_class]))\n            return anchor, pos, neg\n\n        if self.phase == 'test':\n            label = 9\n            labels_ = [5,6,7,8,9]\n            while label in labels_:\n                anchor, label = test_data.__getitem__(index)\n                index = random.randint(0, 10000-1)\n            pos, _ = test_data.__getitem__(random.choice(self.test_indices[label]))\n            all_classes = list(range(5))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = test_data.__getitem__(random.choice(self.test_indices[diff_class]))\n            return anchor, pos, neg\n            \n    def __len__(self):\n        if self.phase == 'train': return 60000\n        return 9000","metadata":{"execution":{"iopub.status.busy":"2021-10-08T14:52:03.721771Z","iopub.execute_input":"2021-10-08T14:52:03.722064Z","iopub.status.idle":"2021-10-08T14:52:03.745029Z","shell.execute_reply.started":"2021-10-08T14:52:03.722037Z","shell.execute_reply":"2021-10-08T14:52:03.743932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_train_data = tripletMNIST5(phase='train')\ntriplet_test_data = tripletMNIST5(phase='test')\n\ntrain_loader = DataLoader(dataset=triplet_train_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=triplet_test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-08T14:52:04.229811Z","iopub.execute_input":"2021-10-08T14:52:04.230092Z","iopub.status.idle":"2021-10-08T14:52:07.038473Z","shell.execute_reply.started":"2021-10-08T14:52:04.230064Z","shell.execute_reply":"2021-10-08T14:52:07.037563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletNet(nn.Module):\n    def __init__(self):\n        super(TripletNet,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 2)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self, anchor, pos, neg):\n        anchor_embd = self.conv1(anchor)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn1(anchor_embd)\n        anchor_embd = self.conv2(anchor_embd)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn2(anchor_embd)\n        anchor_embd = anchor_embd.view(anchor_embd.size()[0], -1)\n        anchor_embd = self.fc1(anchor_embd)\n\n        pos_embd = self.conv1(pos)\n        pos_embd = self.maxpool(pos_embd)\n        pos_embd = self.relu(pos_embd)\n        pos_embd = self.bn1(pos_embd)\n        pos_embd = self.conv2(pos_embd)\n        pos_embd = self.maxpool(pos_embd)\n        pos_embd = self.relu(pos_embd)\n        pos_embd = self.bn2(pos_embd)\n        pos_embd = pos_embd.view(pos_embd.size()[0], -1)\n        pos_embd = self.fc1(pos_embd)\n\n        neg_embd = self.conv1(neg)\n        neg_embd = self.maxpool(neg_embd)\n        neg_embd = self.relu(neg_embd)\n        neg_embd = self.bn1(neg_embd)\n        neg_embd = self.conv2(neg_embd)\n        neg_embd = self.maxpool(neg_embd)\n        neg_embd = self.relu(neg_embd)\n        neg_embd = self.bn2(neg_embd)\n        neg_embd = neg_embd.view(neg_embd.size()[0], -1)\n        neg_embd = self.fc1(neg_embd)\n\n        return anchor_embd, pos_embd, neg_embd","metadata":{"execution":{"iopub.status.busy":"2021-10-08T14:52:07.040032Z","iopub.execute_input":"2021-10-08T14:52:07.040258Z","iopub.status.idle":"2021-10-08T14:52:07.055769Z","shell.execute_reply.started":"2021-10-08T14:52:07.040233Z","shell.execute_reply":"2021-10-08T14:52:07.054878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_net = TripletNet().to(device)\nloss_function = TripletLoss(1.)\noptimizer = torch.optim.Adam(triplet_net.parameters(), lr=1e-1)","metadata":{"execution":{"iopub.status.busy":"2021-10-08T14:52:07.056831Z","iopub.execute_input":"2021-10-08T14:52:07.057372Z","iopub.status.idle":"2021-10-08T14:52:07.070725Z","shell.execute_reply.started":"2021-10-08T14:52:07.057337Z","shell.execute_reply":"2021-10-08T14:52:07.069891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    triplet_net.train()\n    running_loss_train = 0.\n    for i ,(anchor, pos, neg) in enumerate(train_loader):  \n        anchor = anchor.to(device)\n        pos = pos.to(device)\n        neg = neg.to(device)\n        optimizer.zero_grad()\n        anchor_embd, pos_embd, neg_embd = triplet_net(anchor, pos, neg)\n        loss = loss_function(anchor_embd, pos_embd, neg_embd)\n        loss.backward()\n        optimizer.step()\n        running_loss_train += loss.item() * anchor.size(0)\n    epoch_loss_train = running_loss_train / len(triplet_train_data)\n\n    triplet_net.eval()\n    running_loss_test = 0.\n    for i ,(anchor, pos, neg) in enumerate(test_loader):  \n        anchor = anchor.to(device)\n        pos = pos.to(device)\n        neg = neg.to(device)\n        anchor_embd, pos_embd, neg_embd = triplet_net(anchor, pos, neg)\n        loss = loss_function(anchor_embd, pos_embd, neg_embd)\n        running_loss_test += loss.item() * anchor.size(0)\n    epoch_loss_test = running_loss_test / len(triplet_test_data)\n\n    print(f'Epoch {epoch+1}/{num_epochs} | '\n        f'Train loss: {epoch_loss_train} | '\n        f'Test loss: {epoch_loss_test}')","metadata":{"execution":{"iopub.status.busy":"2021-10-08T14:52:21.307397Z","iopub.execute_input":"2021-10-08T14:52:21.307661Z","iopub.status.idle":"2021-10-08T15:04:43.458763Z","shell.execute_reply.started":"2021-10-08T14:52:21.307635Z","shell.execute_reply":"2021-10-08T15:04:43.457822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletNetTest(nn.Module):\n    def __init__(self):\n        super(TripletNetTest,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 2)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self, anchor):\n        anchor_embd = self.conv1(anchor)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn1(anchor_embd)\n        anchor_embd = self.conv2(anchor_embd)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn2(anchor_embd)\n        anchor_embd = anchor_embd.view(anchor_embd.size()[0], -1)\n        anchor_embd = self.fc1(anchor_embd)\n        return anchor_embd","metadata":{"execution":{"iopub.status.busy":"2021-10-08T15:04:43.46064Z","iopub.execute_input":"2021-10-08T15:04:43.460898Z","iopub.status.idle":"2021-10-08T15:04:43.472297Z","shell.execute_reply.started":"2021-10-08T15:04:43.460869Z","shell.execute_reply":"2021-10-08T15:04:43.471256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_test_net = TripletNetTest().to(device)\ntriplet_test_net.load_state_dict(triplet_net.state_dict())\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=len(train_data))\ntest_loader = DataLoader(dataset=test_data, batch_size=len(test_data))\ntriplet_test_net.eval()\n\nwith torch.no_grad():\n    images, train_labels = next(iter(train_loader))\n    train_features = triplet_test_net(images.to(device))\n    train_features = train_features.cpu()\n    \n    images, test_labels = next(iter(test_loader))\n    test_features = triplet_test_net(images.to(device))\n    test_features = test_features.cpu()\n\n\nfigure(num=None, figsize=(15, 12), dpi=80, facecolor='w', edgecolor='k')\nplt.subplot(221)\nfor i in range(5): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(222)\nfor i in range(5): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i], s=1, label=f'{i}')\n\nplt.subplot(223)\nfor i in range(5, 10): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(224)\nfor i in range(5, 10): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i], s=1, label=f'{i}')\n    \n    \nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-10-08T15:05:28.01451Z","iopub.execute_input":"2021-10-08T15:05:28.014806Z","iopub.status.idle":"2021-10-08T15:05:40.103113Z","shell.execute_reply.started":"2021-10-08T15:05:28.014775Z","shell.execute_reply":"2021-10-08T15:05:40.102485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.subplot(121)\nfor i in range(10): plt.scatter(train_features[train_labels==i].T[0], train_features[train_labels==i].T[1], c=clr[i], s=.1)\n\nplt.subplot(122)\nfor i in range(10): plt.scatter(test_features[test_labels==i].T[0], test_features[test_labels==i].T[1], c=clr[i], s=1, label=f'{i}')\n\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-10-08T15:05:40.104458Z","iopub.execute_input":"2021-10-08T15:05:40.104788Z","iopub.status.idle":"2021-10-08T15:05:42.187059Z","shell.execute_reply.started":"2021-10-08T15:05:40.104761Z","shell.execute_reply":"2021-10-08T15:05:42.18616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5): print(train_features[train_labels==i].T[0].mean(), train_features[train_labels==i].T[1].mean())","metadata":{"execution":{"iopub.status.busy":"2021-10-08T15:25:08.747849Z","iopub.execute_input":"2021-10-08T15:25:08.748285Z","iopub.status.idle":"2021-10-08T15:25:08.781033Z","shell.execute_reply.started":"2021-10-08T15:25:08.748242Z","shell.execute_reply":"2021-10-08T15:25:08.780124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(triplet_net.state_dict(), 'five_class_triplet.pt')","metadata":{"execution":{"iopub.status.busy":"2021-10-08T15:26:53.82206Z","iopub.execute_input":"2021-10-08T15:26:53.822433Z","iopub.status.idle":"2021-10-08T15:26:53.830278Z","shell.execute_reply.started":"2021-10-08T15:26:53.82238Z","shell.execute_reply":"2021-10-08T15:26:53.829309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class tripletMNIST(Dataset):\n    def __init__(self, phase):\n        self.samples = []\n        self.phase = phase\n        self.train_indices = []\n        self.test_indices = []\n        for i in range(10):\n            self.train_indices.append([x for x, item in enumerate(train_data.targets == i) if item])\n            self.test_indices.append([x for x, item in enumerate(test_data.targets == i) if item])\n\n    def __getitem__(self, index):\n        if self.phase == 'train':\n            anchor, label = train_data.__getitem__(index) ## __getitem__ of MNIST dataset class\n            pos, _ = train_data.__getitem__(random.choice(self.train_indices[label]))\n            all_classes = list(range(10))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = train_data.__getitem__(random.choice(self.train_indices[diff_class]))\n            return anchor, pos, neg\n\n        if self.phase == 'test':\n            anchor, label = test_data.__getitem__(index)\n            pos, _ = test_data.__getitem__(random.choice(self.test_indices[label]))\n            all_classes = list(range(10))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = test_data.__getitem__(random.choice(self.test_indices[diff_class]))\n            return anchor, pos, neg\n            \n    def __len__(self):\n        if self.phase == 'train': return 60000\n        return 10000\n\n    \n\nclass tripletMNIST5(Dataset):\n    def __init__(self, phase):\n        self.samples = []\n        self.phase = phase\n        self.train_indices = []\n        self.test_indices = []\n        for i in range(5):\n            self.train_indices.append([x for x, item in enumerate(train_data.targets == i) if item])\n            self.test_indices.append([x for x, item in enumerate(test_data.targets == i) if item])\n\n    def __getitem__(self, index):\n        if self.phase == 'train':\n            label = 9\n            labels_ = [5,6,7,8,9]\n            while label in labels_:\n                anchor, label = train_data.__getitem__(index) ## __getitem__ of MNIST dataset class\n                index = random.randint(0, 60000-1)\n            pos, _ = train_data.__getitem__(random.choice(self.train_indices[label]))\n            all_classes = list(range(5))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = train_data.__getitem__(random.choice(self.train_indices[diff_class]))\n            return anchor, pos, neg\n\n        if self.phase == 'test':\n            label = 9\n            labels_ = [5,6,7,8,9]\n            while label in labels_:\n                anchor, label = test_data.__getitem__(index)\n                index = random.randint(0, 10000-1)\n            pos, _ = test_data.__getitem__(random.choice(self.test_indices[label]))\n            all_classes = list(range(5))\n            _ = all_classes.pop(label)\n            diff_class = random.choice(all_classes)\n            neg, _ = test_data.__getitem__(random.choice(self.test_indices[diff_class]))\n            return anchor, pos, neg\n            \n    def __len__(self):\n        if self.phase == 'train': return 60000\n        return 9000","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:47:32.890345Z","iopub.execute_input":"2021-10-09T02:47:32.890811Z","iopub.status.idle":"2021-10-09T02:47:32.915628Z","shell.execute_reply.started":"2021-10-09T02:47:32.89077Z","shell.execute_reply":"2021-10-09T02:47:32.914393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_train_data = tripletMNIST5(phase='train')\ntriplet_test_data = tripletMNIST5(phase='test')\n\ntrain_loader = DataLoader(dataset=triplet_train_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=triplet_test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:23:17.0371Z","iopub.execute_input":"2021-10-09T03:23:17.03743Z","iopub.status.idle":"2021-10-09T03:23:19.978245Z","shell.execute_reply.started":"2021-10-09T03:23:17.037399Z","shell.execute_reply":"2021-10-09T03:23:19.977311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletNet(nn.Module):\n    def __init__(self):\n        super(TripletNet,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 8)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self, anchor, pos, neg):\n        anchor_embd = self.conv1(anchor)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn1(anchor_embd)\n        anchor_embd = self.conv2(anchor_embd)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn2(anchor_embd)\n        anchor_embd = anchor_embd.view(anchor_embd.size()[0], -1)\n        anchor_embd = self.fc1(anchor_embd)\n\n        pos_embd = self.conv1(pos)\n        pos_embd = self.maxpool(pos_embd)\n        pos_embd = self.relu(pos_embd)\n        pos_embd = self.bn1(pos_embd)\n        pos_embd = self.conv2(pos_embd)\n        pos_embd = self.maxpool(pos_embd)\n        pos_embd = self.relu(pos_embd)\n        pos_embd = self.bn2(pos_embd)\n        pos_embd = pos_embd.view(pos_embd.size()[0], -1)\n        pos_embd = self.fc1(pos_embd)\n\n        neg_embd = self.conv1(neg)\n        neg_embd = self.maxpool(neg_embd)\n        neg_embd = self.relu(neg_embd)\n        neg_embd = self.bn1(neg_embd)\n        neg_embd = self.conv2(neg_embd)\n        neg_embd = self.maxpool(neg_embd)\n        neg_embd = self.relu(neg_embd)\n        neg_embd = self.bn2(neg_embd)\n        neg_embd = neg_embd.view(neg_embd.size()[0], -1)\n        neg_embd = self.fc1(neg_embd)\n\n        return anchor_embd, pos_embd, neg_embd","metadata":{"execution":{"iopub.status.busy":"2021-10-09T02:47:52.214884Z","iopub.execute_input":"2021-10-09T02:47:52.215257Z","iopub.status.idle":"2021-10-09T02:47:52.230961Z","shell.execute_reply.started":"2021-10-09T02:47:52.215222Z","shell.execute_reply":"2021-10-09T02:47:52.229724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_net = TripletNet().to(device)\nloss_function = TripletLoss(1.)\noptimizer = torch.optim.Adam(triplet_net.parameters(), lr=1e-1)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:23:24.942808Z","iopub.execute_input":"2021-10-09T03:23:24.943993Z","iopub.status.idle":"2021-10-09T03:23:24.952495Z","shell.execute_reply.started":"2021-10-09T03:23:24.943926Z","shell.execute_reply":"2021-10-09T03:23:24.951316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    triplet_net.train()\n    running_loss_train = 0.\n    for i ,(anchor, pos, neg) in enumerate(train_loader):  \n        anchor = anchor.to(device)\n        pos = pos.to(device)\n        neg = neg.to(device)\n        optimizer.zero_grad()\n        anchor_embd, pos_embd, neg_embd = triplet_net(anchor, pos, neg)\n        loss = loss_function(anchor_embd, pos_embd, neg_embd)\n        loss.backward()\n        optimizer.step()\n        running_loss_train += loss.item() * anchor.size(0)\n    epoch_loss_train = running_loss_train / len(triplet_train_data)\n\n    triplet_net.eval()\n    running_loss_test = 0.\n    for i ,(anchor, pos, neg) in enumerate(test_loader):  \n        anchor = anchor.to(device)\n        pos = pos.to(device)\n        neg = neg.to(device)\n        anchor_embd, pos_embd, neg_embd = triplet_net(anchor, pos, neg)\n        loss = loss_function(anchor_embd, pos_embd, neg_embd)\n        running_loss_test += loss.item() * anchor.size(0)\n    epoch_loss_test = running_loss_test / len(triplet_test_data)\n\n    print(f'Epoch {epoch+1}/{num_epochs} | '\n        f'Train loss: {epoch_loss_train} | '\n        f'Test loss: {epoch_loss_test}')","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:23:25.172612Z","iopub.execute_input":"2021-10-09T03:23:25.173826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletNetTest(nn.Module):\n    def __init__(self):\n        super(TripletNetTest,self).__init__()\n        self.conv1 = nn.Conv2d(1, 8, 5)\n        self.maxpool = nn.MaxPool2d(2, stride=2)\n        self.conv2 = nn.Conv2d(8, 8, 5)\n        self.fc1 = nn.Linear(8*4*4, 8)\n        self.relu = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(8)\n        self.bn2 = nn.BatchNorm2d(8)\n\n    def forward(self, anchor):\n        anchor_embd = self.conv1(anchor)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn1(anchor_embd)\n        anchor_embd = self.conv2(anchor_embd)\n        anchor_embd = self.maxpool(anchor_embd)\n        anchor_embd = self.relu(anchor_embd)\n        anchor_embd = self.bn2(anchor_embd)\n        anchor_embd = anchor_embd.view(anchor_embd.size()[0], -1)\n        anchor_embd = self.fc1(anchor_embd)\n        return anchor_embd","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:12:19.220277Z","iopub.execute_input":"2021-10-09T03:12:19.22159Z","iopub.status.idle":"2021-10-09T03:12:19.232611Z","shell.execute_reply.started":"2021-10-09T03:12:19.22154Z","shell.execute_reply":"2021-10-09T03:12:19.231935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"triplet_test_net = TripletNetTest().to(device)\ntriplet_test_net.load_state_dict(triplet_net.state_dict())\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=len(train_data))\ntest_loader = DataLoader(dataset=test_data, batch_size=len(test_data))\ntriplet_test_net.eval()\n\nwith torch.no_grad():\n    images, train_labels = next(iter(train_loader))\n    train_features = triplet_test_net(images.to(device))\n    train_features = train_features.cpu()\n    \n    images, test_labels = next(iter(test_loader))\n    test_features = triplet_test_net(images.to(device))\n    test_features = test_features.cpu()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:12:23.39539Z","iopub.execute_input":"2021-10-09T03:12:23.396533Z","iopub.status.idle":"2021-10-09T03:12:43.089776Z","shell.execute_reply.started":"2021-10-09T03:12:23.396484Z","shell.execute_reply":"2021-10-09T03:12:43.089181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.mean(train_features, dim=0)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:13:15.893244Z","iopub.execute_input":"2021-10-09T03:13:15.893536Z","iopub.status.idle":"2021-10-09T03:13:15.901399Z","shell.execute_reply.started":"2021-10-09T03:13:15.893506Z","shell.execute_reply":"2021-10-09T03:13:15.900246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5): print(train_features[train_labels==i].T[0].mean(), train_features[train_labels==i].T[1].mean())","metadata":{"execution":{"iopub.status.busy":"2021-10-08T15:25:08.747849Z","iopub.execute_input":"2021-10-08T15:25:08.748285Z","iopub.status.idle":"2021-10-08T15:25:08.781033Z","shell.execute_reply.started":"2021-10-08T15:25:08.748242Z","shell.execute_reply":"2021-10-08T15:25:08.780124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5): print(torch.mean(train_features[train_labels==i], dim=0))","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:14:16.478664Z","iopub.execute_input":"2021-10-09T03:14:16.479006Z","iopub.status.idle":"2021-10-09T03:14:16.498039Z","shell.execute_reply.started":"2021-10-09T03:14:16.478972Z","shell.execute_reply":"2021-10-09T03:14:16.497161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(triplet_net.state_dict(), 'mnist_triplet_8__.pt')","metadata":{"execution":{"iopub.status.busy":"2021-10-09T03:14:33.148275Z","iopub.execute_input":"2021-10-09T03:14:33.148748Z","iopub.status.idle":"2021-10-09T03:14:33.155587Z","shell.execute_reply.started":"2021-10-09T03:14:33.148701Z","shell.execute_reply":"2021-10-09T03:14:33.154509Z"},"trusted":true},"execution_count":null,"outputs":[]}]}