{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":1099.682295,"end_time":"2024-08-17T11:24:32.360355","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-08-17T11:06:12.678060","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pickle\nfrom PIL import Image\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport os\nimport torch.nn.functional as F\nimport requests\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport seaborn as sn\ntorch.manual_seed(1) #Set manual seed\nfrom sklearn import preprocessing\nfrom torch.utils.data import DataLoader, Dataset, Subset\nimport torch.optim as optim\nfrom sklearn.metrics import roc_auc_score, average_precision_score, balanced_accuracy_score, f1_score, accuracy_score, recall_score, precision_score, roc_curve, auc\nfrom time import time\nfrom sklearn.preprocessing import label_binarize\nimport torchvision.models as models\n# from torchvision.models import resnet10\\\nfrom sklearn.decomposition import PCA\nfrom torchvision.models.resnet import ResNet, Bottleneck\n\n#Define GPU device\nmps_device = torch.device(\"mps\")\n\nimport torch\nimport math\nimport torch.nn as nn\nfrom torch.nn.parameter import Parameter\nimport time\n\nimport pdb\n\nimport math\nimport torch.nn as nn\nimport torch.nn.functional as F\n# from layers.ModulatedAttLayer import ModulatedAttLayer","metadata":{"papermill":{"duration":7.393091,"end_time":"2024-08-17T11:06:22.901191","exception":false,"start_time":"2024-08-17T11:06:15.508100","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-08T14:41:56.356345Z","iopub.execute_input":"2024-10-08T14:41:56.356766Z","iopub.status.idle":"2024-10-08T14:42:02.485529Z","shell.execute_reply.started":"2024-10-08T14:41:56.356722Z","shell.execute_reply":"2024-10-08T14:42:02.484516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preparation","metadata":{"papermill":{"duration":0.008306,"end_time":"2024-08-17T11:06:22.994035","exception":false,"start_time":"2024-08-17T11:06:22.985729","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IMAGE_SIZE = 32\nmean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261]\n# These values are mostly used by researchers as found to very useful in fast convergence\n\n# We can try various transformation for good generalization of model.\ncomposed_train = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), # Resize the image in a 32 x 32 shape\n                                     transforms.RandomRotation(20), # Randomly rotate some images by 20 degrees\n                                     transforms.RandomHorizontalFlip(0.1), #Randomly horizontal flip the images\n                                     transforms.ColorJitter(brightness = 0.1, #Randomly adjust color jitter of the images\n                                                            contrast = 0.1,\n                                                            saturation = 0.1),\n                                     transforms.RandomAdjustSharpness(sharpness_factor = 2,\n                                                                      p = 0.1), # Randomly adjust sharpness\n                                     transforms.ToTensor(), # Converting image to tensor\n                                     transforms.Normalize(mean, std), # Normalizing with standard mean and standard deviation\n                                     transforms.RandomErasing(p = 0.75, scale = (0.02, 0.1), value = 1.0, inplace = False)\n                                    ])\n\ncomposed_test = transforms.Compose([transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize(mean, std)\n                                   ])","metadata":{"papermill":{"duration":0.021008,"end_time":"2024-08-17T11:06:23.023735","exception":false,"start_time":"2024-08-17T11:06:23.002727","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-08T14:42:02.487759Z","iopub.execute_input":"2024-10-08T14:42:02.488472Z","iopub.status.idle":"2024-10-08T14:42:02.497457Z","shell.execute_reply.started":"2024-10-08T14:42:02.488423Z","shell.execute_reply":"2024-10-08T14:42:02.496437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCIFAR10(datasets.CIFAR10):\n    def __getitem__(self, index):\n        img, target = super().__getitem__(index)\n        return img, target, index\n    \n#Custom Collate Function\ndef custom_collate_fn(batch):\n    data = torch.stack([item[0] for item in batch])\n    targets = torch.tensor([item[1] for item in batch])\n    indices = torch.tensor([item[2] for item in batch])\n    return data, targets, indices","metadata":{"papermill":{"duration":0.018155,"end_time":"2024-08-17T11:06:23.050756","exception":false,"start_time":"2024-08-17T11:06:23.032601","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-08T14:42:02.498704Z","iopub.execute_input":"2024-10-08T14:42:02.499009Z","iopub.status.idle":"2024-10-08T14:42:02.517657Z","shell.execute_reply.started":"2024-10-08T14:42:02.498976Z","shell.execute_reply":"2024-10-08T14:42:02.516660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data and transform the dataset\ntrain_dataset = CustomCIFAR10(root='./data', train=True, download=True, transform=composed_train)\nvalidation_dataset = CustomCIFAR10(root='./data', train=False, download=True, transform=composed_test)\n\n# Create train and validation batch for training\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=100, shuffle=True, num_workers=2, collate_fn=custom_collate_fn)\nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_dataset, batch_size=200, shuffle=False, num_workers=2, collate_fn=custom_collate_fn)","metadata":{"papermill":{"duration":16.62251,"end_time":"2024-08-17T11:06:39.682032","exception":false,"start_time":"2024-08-17T11:06:23.059522","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-08T14:42:02.518765Z","iopub.execute_input":"2024-10-08T14:42:02.519101Z","iopub.status.idle":"2024-10-08T14:42:12.094398Z","shell.execute_reply.started":"2024-10-08T14:42:02.519067Z","shell.execute_reply":"2024-10-08T14:42:12.093575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Initialize the classifier**","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass _Classifier(nn.Module):\n    def __init__(self, feat_dim=None, num_classes=None, dtype=None):\n        super().__init__()\n        self.weight = nn.Parameter(torch.empty(num_classes, feat_dim, dtype=dtype))\n        self.weight.data.uniform_(-1, 1).renorm_(2, 0, 1e-5).mul_(1e5)\n\n    @property\n    def dtype(self):\n        return self.weight.dtype\n\n    def forward(self, x):\n        raise NotImplementedError\n\n    def apply_weight(self, weight):\n        self.weight.data = weight.clone()\n    \n\nclass LinearClassifier(_Classifier):\n    def __init__(self, feat_dim=None, num_classes=None, dtype=None, **kwargs):\n        super().__init__(feat_dim, num_classes, dtype)\n        nn.init.kaiming_normal_(self.weight.data)\n        self.bias = nn.Parameter(torch.zeros(num_classes, dtype=dtype))\n\n    def forward(self, x):\n        return F.linear(x, self.weight, self.bias)\n\n\nclass CosineClassifier(_Classifier):\n    def __init__(self, feat_dim=None, num_classes=None, dtype=None, scale=30, **kwargs):\n        super().__init__(feat_dim, num_classes, dtype)\n        self.scale = scale\n\n    def forward(self, x):\n        x = F.normalize(x, dim=-1)\n        weight = F.normalize(self.weight, dim=-1)\n        return F.linear(x, weight) * self.scale\n\n\nclass L2NormedClassifier(_Classifier):\n    def __init__(self, feat_dim=None, num_classes=None, dtype=None, **kwargs):\n        super().__init__(feat_dim, num_classes, dtype)\n    \n    def forward(self, x):\n        weight = F.normalize(self.weight, dim=-1)\n        return F.linear(x, weight)\n\n\nclass LayerNormedClassifier(_Classifier):\n    def __init__(self, feat_dim=None, num_classes=None, dtype=None, **kwargs):\n        super().__init__(feat_dim, num_classes, dtype)\n        self.ln = nn.LayerNorm(feat_dim, elementwise_affine=False, eps=1e-12, dtype=dtype)\n\n    def forward(self, x):\n        x = self.ln(x)\n        weight = F.normalize(self.weight, dim=-1)\n        return F.linear(x, weight)","metadata":{"papermill":{"duration":0.029276,"end_time":"2024-08-17T11:24:30.503482","exception":false,"start_time":"2024-08-17T11:24:30.474206","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-08T14:42:12.097845Z","iopub.execute_input":"2024-10-08T14:42:12.098154Z","iopub.status.idle":"2024-10-08T14:42:12.113540Z","shell.execute_reply.started":"2024-10-08T14:42:12.098122Z","shell.execute_reply":"2024-10-08T14:42:12.112495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming the feature dimension after your backbone model is 512 (example)\nfeat_dim = 512\nnum_classes = 10  # CIFAR-10 has 10 classes\n\n# Initialize the classifier of your choice\nclassifier = LinearClassifier(feat_dim=feat_dim, num_classes=num_classes, dtype=torch.float32)\n# Alternatively, you can use any of the other classifiers:\n# classifier = CosineClassifier(feat_dim=feat_dim, num_classes=num_classes, dtype=torch.float32, scale=30)\n# classifier = L2NormedClassifier(feat_dim=feat_dim, num_classes=num_classes, dtype=torch.float32)\n# classifier = LayerNormedClassifier(feat_dim=feat_dim, num_classes=num_classes, dtype=torch.float32)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.114643Z","iopub.execute_input":"2024-10-08T14:42:12.114936Z","iopub.status.idle":"2024-10-08T14:42:12.190092Z","shell.execute_reply.started":"2024-10-08T14:42:12.114898Z","shell.execute_reply":"2024-10-08T14:42:12.189164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Intergrate the classifer with the model**","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch\nimport torch.nn.init as init\n\ndef _weights_init(m):\n    classname = m.__class__.__name__\n    if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):\n        init.kaiming_normal_(m.weight)\n\n\nimport torch.nn.functional as F\n\nclass LambdaLayer(nn.Module):\n\n    def __init__(self, lambd):\n        super(LambdaLayer, self).__init__()\n        self.lambd = lambd\n\n    def forward(self, x):\n        return self.lambd(x)\n\nclass BasicBlock_s(nn.Module):\n    expansion = 1\n\n    def __init__(self, in_planes, planes, stride=1, option='A'):\n        super(BasicBlock_s, self).__init__()\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != planes:\n            if option == 'A':\n                \"\"\"\n                For CIFAR10 ResNet paper uses option A.\n                \"\"\"\n                self.shortcut = LambdaLayer(lambda x:\n                                            F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes // 4, planes // 4), \"constant\",\n                                                  0))\n            elif option == 'B':\n                self.shortcut = nn.Sequential(\n                    nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),\n                    nn.BatchNorm2d(self.expansion * planes)\n                )\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        out += self.shortcut(x)\n        out = F.relu(out)\n        return out\n\nclass ResNet_modify(nn.Module):\n\n    def __init__(self, block, num_blocks, num_classes=100, nf=64):\n        super(ResNet_modify, self).__init__()\n        self.in_planes = nf\n\n        self.conv1 = nn.Conv2d(3, self.in_planes, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(self.in_planes)\n        self.layer1 = self._make_layer(block, 1 * nf, num_blocks[0], stride=1)\n        self.layer2 = self._make_layer(block, 2 * nf, num_blocks[1], stride=2)\n        self.layer3 = self._make_layer(block, 4 * nf, num_blocks[2], stride=2)\n        self.out_dim = 4 * nf * block.expansion\n\n        self.fc = nn.Linear(self.out_dim, num_classes)\n        # self.fc_cb = torch.nn.utils.weight_norm(nn.Linear(512 * block.expansion, num_class), dim=0)\n        hidden_dim = 128\n        self.fc_cb = nn.Linear(self.out_dim, num_classes)\n        self.contrast_head = nn.Sequential(\n            nn.Linear(hidden_dim, hidden_dim),\n        )\n        self.projection_head = nn.Sequential(\n            nn.Linear(self.out_dim, hidden_dim),\n\n        )\n        self.apply(_weights_init)\n\n    def _make_layer(self, block, planes, num_blocks, stride):\n        strides = [stride] + [1] * (num_blocks - 1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, stride))\n            self.in_planes = planes * block.expansion\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x, train=False):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.avg_pool2d(out, out.size()[3])\n        feature = out.view(out.size(0), -1)\n\n        if train is True:\n            out = self.fc(feature)\n            out_cb = self.fc_cb(feature)\n            z = self.projection_head(feature)\n            p = self.contrast_head(z)\n            return out, out_cb, z, p\n        else:\n            out = self.fc_cb(feature)\n\n            return out\n\n\nclass BasicBlock(nn.Module):\n    \"\"\"Basic Block for resnet 18 and resnet 34\n    \"\"\"\n    # BasicBlock and BottleNeck block\n    # have different output size\n    # we use class attribute expansion\n    # to distinct\n    expansion = 1\n\n    def __init__(self, in_channels, out_channels, stride=1):\n        super().__init__()\n\n        # residual function\n        self.residual_function = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels * BasicBlock.expansion, kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels * BasicBlock.expansion)\n        )\n\n        # shortcut\n        self.shortcut = nn.Sequential()\n\n        # the shortcut output dimension is not the same with residual function\n        # use 1*1 convolution to match the dimension\n        if stride != 1 or in_channels != BasicBlock.expansion * out_channels:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels * BasicBlock.expansion, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels * BasicBlock.expansion)\n            )\n\n    def forward(self, x):\n        return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x))\n\n\nclass BottleNeck(nn.Module):\n    \"\"\"Residual block for resnet over 50 layers\n\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, in_channels, out_channels, stride=1):\n        super().__init__()\n        self.residual_function = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels * BottleNeck.expansion, kernel_size=1, bias=False),\n            nn.BatchNorm2d(out_channels * BottleNeck.expansion),\n        )\n\n        self.shortcut = nn.Sequential()\n\n        if stride != 1 or in_channels != out_channels * BottleNeck.expansion:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels * BottleNeck.expansion, stride=stride, kernel_size=1, bias=False),\n                nn.BatchNorm2d(out_channels * BottleNeck.expansion)\n            )\n\n    def forward(self, x):\n        return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x))\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, num_block, num_class=100):\n        super().__init__()\n        self.in_channels = 64\n        self.num_class = num_class\n\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True))\n        # we use a different inputsize than the original paper\n        # so conv2_x's stride is 1\n\n        self.conv2_x = self._make_layer(block, 64, num_block[0], 1)\n        self.conv3_x = self._make_layer(block, 128, num_block[1], 2)\n        self.conv4_x = self._make_layer(block, 256, num_block[2], 2)\n        self.conv5_x = self._make_layer(block, 512, num_block[3], 2)\n        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\n        # from torch.nn import w\n\n        self.fc = nn.Linear(512 * block.expansion, num_class)\n        # self.fc_cb = torch.nn.utils.weight_norm(nn.Linear(512 * block.expansion, num_class), dim=0)\n        hidden_dim=256\n        self.fc_cb = nn.Linear(512 * block.expansion, num_class)\n        self.contrast_head = nn.Sequential(\n            nn.Linear(hidden_dim, hidden_dim),\n        )\n        self.projection_head = nn.Sequential(\n            nn.Linear(512 * block.expansion, hidden_dim),\n        )\n\n    def _make_layer(self, block, out_channels, num_blocks, stride):\n        \"\"\"make resnet layers(by layer i didnt mean this 'layer' was the\n        same as a neuron netowork layer, ex. conv layer), one layer may\n        contain more than one residual block\n        Args:\n            block: block type, basic block or bottle neck block\n            out_channels: output depth channel number of this layer\n            num_blocks: how many blocks per layer\n            stride: the stride of the first block of this layer\n\n        Return:\n            return a resnet layer\n        \"\"\"\n        # we have num_block blocks per layer, the first block\n        # could be 1 or 2, other blocks would always be 1\n        strides = [stride] + [1] * (num_blocks - 1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.in_channels, out_channels, stride))\n            self.in_channels = out_channels * block.expansion\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x, train=False):\n        output = self.conv1(x)\n        output = self.conv2_x(output)\n        output = self.conv3_x(output)\n        output = self.conv4_x(output)\n        output = self.conv5_x(output)\n        output = self.avg_pool(output)\n        feature = output.view(output.size(0), -1)\n        if train is True:\n            out = self.fc(feature)\n            out_cb = self.fc_cb(feature)\n            z = self.projection_head(feature)\n            p = self.contrast_head(z)\n            return out, out_cb, z,p\n        else:\n            out = self.fc_cb(feature)\n            return out\n\n\ndef resnet18(num_class=100):\n    \"\"\" return a ResNet 18 object\n    \"\"\"\n    return ResNet(BasicBlock, [2, 2, 2, 2], num_class=num_class)\n\ndef resnet32(num_class=10):\n    return ResNet_modify(BasicBlock_s, [5, 5, 5], num_classes=num_class)\n\ndef resnet34(num_class=100):\n    \"\"\" return a ResNet 34 object\n    \"\"\"\n    return ResNet(BasicBlock, [3, 4, 6, 3], num_class=num_class)\n\n\ndef resnet50(num_class=100):\n    \"\"\" return a ResNet 50 object\n    \"\"\"\n    return ResNet(BottleNeck, [3, 4, 6, 3], num_class=num_class)\n\n\ndef resnet101(num_class=100):\n    \"\"\" return a ResNet 101 object\n    \"\"\"\n    return ResNet(BottleNeck, [3, 4, 23, 3], num_class=num_class)\n\n\ndef resnet152(num_class=100):\n    \"\"\" return a ResNet 152 object\n    \"\"\"\n    return ResNet(BottleNeck, [3, 8, 36, 3], num_class=num_class)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.191580Z","iopub.execute_input":"2024-10-08T14:42:12.192184Z","iopub.status.idle":"2024-10-08T14:42:12.238772Z","shell.execute_reply.started":"2024-10-08T14:42:12.192137Z","shell.execute_reply":"2024-10-08T14:42:12.237884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming you want to use ResNet18\n# model = resnet18(num_class=100)\n\n# If you want to use a different ResNet version, simply replace with the appropriate function:\nmodel = resnet32(num_class=10)\n# model = resnet34(num_class=100)\n# model = resnet50(num_class=100)\n# model = resnet101(num_class=100)\n# model = resnet152(num_class=100)\n\n# Now you can use this model for training or inference\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.239761Z","iopub.execute_input":"2024-10-08T14:42:12.240049Z","iopub.status.idle":"2024-10-08T14:42:12.376083Z","shell.execute_reply.started":"2024-10-08T14:42:12.240017Z","shell.execute_reply":"2024-10-08T14:42:12.375052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training and Evaluation**","metadata":{}},{"cell_type":"code","source":"# # Define loss function and optimizer\n# criterion = nn.CrossEntropyLoss()\n# optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n# # Example training loop\n# for epoch in range(10):\n#     model.train()\n#     for images, labels, _ in train_loader:\n#         optimizer.zero_grad()\n#         outputs = model(images)\n#         loss = criterion(outputs, labels)\n#         loss.backward()\n#         optimizer.step()\n    \n#     # Evaluate on validation set\n#     model.eval()\n#     with torch.no_grad():\n#         for images, labels, _ in validation_loader:\n#             outputs = model(images)\n#             # Compute validation metrics...","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.377522Z","iopub.execute_input":"2024-10-08T14:42:12.378314Z","iopub.status.idle":"2024-10-08T14:42:12.383148Z","shell.execute_reply.started":"2024-10-08T14:42:12.378266Z","shell.execute_reply":"2024-10-08T14:42:12.382215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\n\ndef focal_loss(input_values, gamma):\n    \"\"\"Computes the focal loss\"\"\"\n    p = torch.exp(-input_values)\n    loss = (1 - p) ** gamma * input_values\n    return loss.mean()\n\nclass FocalLoss(nn.Module):\n    def __init__(self, weight=None, gamma=2.0):\n        super().__init__()\n        assert gamma >= 0\n        self.gamma = gamma\n        self.weight = weight\n\n    def forward(self, logit, target):\n        return focal_loss(F.cross_entropy(logit, target, reduction='none', weight=self.weight), self.gamma)\n\n\nclass LDAMLoss(nn.Module):\n    def __init__(self, cls_num_list, max_m=0.5, s=30):\n        super().__init__()\n        m_list = 1.0 / torch.sqrt(torch.sqrt(cls_num_list))\n        m_list = m_list * (max_m / torch.max(m_list))\n        m_list = torch.cuda.FloatTensor(m_list)\n        self.m_list = m_list\n        self.s = s\n\n    def forward(self, logit, target):\n        index = torch.zeros_like(logit, dtype=torch.bool)\n        index.scatter_(1, target.data.view(-1, 1), 1)\n\n        index_float = index.type(torch.cuda.FloatTensor)\n        batch_m = torch.matmul(self.m_list[None, :], index_float.transpose(0, 1))\n        batch_m = batch_m.view((-1, 1))\n        logit_m = logit - batch_m * self.s  # scale only the margin, as the logit is already scaled.\n\n        output = torch.where(index, logit_m, logit)\n        return F.cross_entropy(output, target)\n\n\nclass ClassBalancedLoss(nn.Module):\n    def __init__(self, cls_num_list, beta=0.9999):\n        super().__init__()\n        per_cls_weights = (1.0 - beta) / (1.0 - (beta ** cls_num_list))\n        per_cls_weights = per_cls_weights / torch.mean(per_cls_weights)\n        self.per_cls_weights = per_cls_weights\n    \n    def forward(self, logit, target):\n        logit = logit.to(self.per_cls_weights.dtype)\n        return F.cross_entropy(logit, target, weight=self.per_cls_weights)\n\n\nclass GeneralizedReweightLoss(nn.Module):\n    def __init__(self, cls_num_list, exp_scale=1.0):\n        super().__init__()\n        cls_num_ratio = cls_num_list / torch.sum(cls_num_list)\n        per_cls_weights = 1.0 / (cls_num_ratio ** exp_scale)\n        per_cls_weights = per_cls_weights / torch.mean(per_cls_weights)\n        self.per_cls_weights = per_cls_weights\n    \n    def forward(self, logit, target):\n        logit = logit.to(self.per_cls_weights.dtype)\n        return F.cross_entropy(logit, target, weight=self.per_cls_weights)\n\n\nclass BalancedSoftmaxLoss(nn.Module):\n    def __init__(self, cls_num_list):\n        super().__init__()\n        cls_num_ratio = cls_num_list / torch.sum(cls_num_list)\n        log_cls_num = torch.log(cls_num_ratio)\n        self.log_cls_num = log_cls_num\n\n    def forward(self, logit, target):\n        logit_adjusted = logit + self.log_cls_num.unsqueeze(0)\n        return F.cross_entropy(logit_adjusted, target)\n\n\nclass LogitAdjustedLoss(nn.Module):\n    def __init__(self, cls_num_list, tau=1.0):\n        super().__init__()\n        cls_num_ratio = cls_num_list / torch.sum(cls_num_list)\n        log_cls_num = torch.log(cls_num_ratio)\n        self.log_cls_num = log_cls_num\n        self.tau = tau\n\n    def forward(self, logit, target):\n        logit_adjusted = logit + self.tau * self.log_cls_num.unsqueeze(0)\n        return F.cross_entropy(logit_adjusted, target)\n\n\nclass LADELoss(nn.Module):\n    def __init__(self, cls_num_list, remine_lambda=0.1, estim_loss_weight=0.1):\n        super().__init__()\n        self.num_classes = len(cls_num_list)\n        self.prior = cls_num_list / torch.sum(cls_num_list)\n\n        self.balanced_prior = torch.tensor(1. / self.num_classes).float().to(self.prior.device)\n        self.remine_lambda = remine_lambda\n\n        self.cls_weight = cls_num_list / torch.sum(cls_num_list)\n        self.estim_loss_weight = estim_loss_weight\n\n    def mine_lower_bound(self, x_p, x_q, num_samples_per_cls):\n        N = x_p.size(-1)\n        first_term = torch.sum(x_p, -1) / (num_samples_per_cls + 1e-8)\n        second_term = torch.logsumexp(x_q, -1) - np.log(N)\n \n        return first_term - second_term, first_term, second_term\n\n    def remine_lower_bound(self, x_p, x_q, num_samples_per_cls):\n        loss, first_term, second_term = self.mine_lower_bound(x_p, x_q, num_samples_per_cls)\n        reg = (second_term ** 2) * self.remine_lambda\n        return loss - reg, first_term, second_term\n\n    def forward(self, logit, target):\n        logit_adjusted = logit + torch.log(self.prior).unsqueeze(0)\n        ce_loss =  F.cross_entropy(logit_adjusted, target)\n\n        per_cls_pred_spread = logit.T * (target == torch.arange(0, self.num_classes).view(-1, 1).type_as(target))  # C x N\n        pred_spread = (logit - torch.log(self.prior + 1e-9) + torch.log(self.balanced_prior + 1e-9)).T  # C x N\n\n        num_samples_per_cls = torch.sum(target == torch.arange(0, self.num_classes).view(-1, 1).type_as(target), -1).float()  # C\n        estim_loss, first_term, second_term = self.remine_lower_bound(per_cls_pred_spread, pred_spread, num_samples_per_cls)\n        estim_loss = -torch.sum(estim_loss * self.cls_weight)\n\n        return ce_loss + self.estim_loss_weight * estim_loss","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.384682Z","iopub.execute_input":"2024-10-08T14:42:12.385451Z","iopub.status.idle":"2024-10-08T14:42:12.413729Z","shell.execute_reply.started":"2024-10-08T14:42:12.385382Z","shell.execute_reply":"2024-10-08T14:42:12.412950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate class distribution as NumPy array\ncls_num_list = np.bincount(train_dataset.targets)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Convert to PyTorch tensor and move to the appropriate device (GPU/CPU)\ncls_num_list = torch.tensor(cls_num_list, dtype=torch.float32).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.414998Z","iopub.execute_input":"2024-10-08T14:42:12.415275Z","iopub.status.idle":"2024-10-08T14:42:12.602284Z","shell.execute_reply.started":"2024-10-08T14:42:12.415244Z","shell.execute_reply":"2024-10-08T14:42:12.601497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = model.to(device)\ncriterion = BalancedSoftmaxLoss(cls_num_list=cls_num_list).to(device)  # Replace with any other custom loss function\noptimizer = optim.Adam(model.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.603586Z","iopub.execute_input":"2024-10-08T14:42:12.604222Z","iopub.status.idle":"2024-10-08T14:42:12.711289Z","shell.execute_reply.started":"2024-10-08T14:42:12.604176Z","shell.execute_reply":"2024-10-08T14:42:12.710526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop\nnum_epochs = 10\nfor epoch in range(num_epochs):\n    model.train()  # Set the model to training mode\n    running_loss = 0.0  # Track the running loss for the epoch\n    \n    for inputs, targets, _ in train_loader:\n        # Move inputs and targets to the correct device (GPU or CPU)\n        inputs, targets = inputs.to(device), targets.to(device)\n        \n        # Forward pass\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        \n        # Backward pass and optimization\n        optimizer.zero_grad()  # Zero the gradients\n        loss.backward()  # Backpropagation\n        optimizer.step()  # Update the weights\n        \n        # Accumulate the loss\n        running_loss += loss.item()\n    \n    # Average loss for the epoch\n    avg_loss = running_loss / len(train_loader)\n    \n    print(f'Epoch {epoch+1}/{num_epochs}, Loss: {avg_loss:.4f}')\n\n# Optional: Include validation accuracy or loss calculation after each epoch\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:42:12.712278Z","iopub.execute_input":"2024-10-08T14:42:12.712600Z","iopub.status.idle":"2024-10-08T14:50:11.852756Z","shell.execute_reply.started":"2024-10-08T14:42:12.712566Z","shell.execute_reply":"2024-10-08T14:50:11.851537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\nmodel.eval()  # Set the model to evaluation mode\nall_preds = []\nall_targets = []\n\nwith torch.no_grad():\n    for inputs, targets, _ in validation_loader:\n        # Move inputs and targets to the correct device\n        inputs, targets = inputs.to(device), targets.to(device)\n        \n        # Forward pass to get outputs\n        outputs = model(inputs)\n        \n        # Get predictions by finding the class with max probability (logits)\n        _, preds = torch.max(outputs, 1)\n        \n        # Move predictions and targets to CPU and convert to numpy arrays\n        all_preds.extend(preds.cpu().numpy())\n        all_targets.extend(targets.cpu().numpy())\n\n# Convert lists to numpy arrays\nall_preds = np.array(all_preds)\nall_targets = np.array(all_targets)\n\n# Calculate evaluation metrics\naccuracy = accuracy_score(all_targets, all_preds)\nbalanced_acc = balanced_accuracy_score(all_targets, all_preds)\nprecision = precision_score(all_targets, all_preds, average='macro')\nf1 = f1_score(all_targets, all_preds, average='macro')\nrecall = recall_score(all_targets, all_preds, average='macro')\n\n# Print the metrics\nprint(f'Accuracy: {accuracy:.4f}')\nprint(f'Balanced Accuracy: {balanced_acc:.4f}')\nprint(f'Precision: {precision:.4f}')\nprint(f'F1-score: {f1:.4f}')\nprint(f'Recall: {recall:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:50:11.857129Z","iopub.execute_input":"2024-10-08T14:50:11.857581Z","iopub.status.idle":"2024-10-08T14:50:15.499552Z","shell.execute_reply.started":"2024-10-08T14:50:11.857541Z","shell.execute_reply":"2024-10-08T14:50:15.498405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.preprocessing import label_binarize\n\n# # Assuming 10 output classes (adjust if different)\n# n_classes = model.fc3.out_features\n\n# # Binarize the output labels (one-hot encode targets and predictions)\n# all_targets_bin = label_binarize(all_targets, classes=np.arange(n_classes))\n# all_preds_bin = label_binarize(all_preds, classes=np.arange(n_classes))\n\n# # Calculate ROC curve and AUC for each class\n# fpr = {}\n# tpr = {}\n# roc_auc = {}\n\n# for i in range(n_classes):\n#     fpr[i], tpr[i], _ = roc_curve(all_targets_bin[:, i], all_preds_bin[:, i])\n#     roc_auc[i] = auc(fpr[i], tpr[i])\n\n# # Plot ROC curve for each class\n# plt.figure(figsize=(10, 8))\n# for i in range(n_classes):\n#     plt.plot(fpr[i], tpr[i], label=f'Class {i} (AUC = {roc_auc[i]:.2f})')\n\n# # Plot the diagonal line representing random chance\n# plt.plot([0, 1], [0, 1], 'k--')\n\n# # Set plot limits and labels\n# plt.xlim([0.0, 1.0])\n# plt.ylim([0.0, 1.05])\n# plt.xlabel('False Positive Rate')\n# plt.ylabel('True Positive Rate')\n# plt.title('ROC Curve for Each Class')\n# plt.legend(loc='lower right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T14:50:15.501681Z","iopub.execute_input":"2024-10-08T14:50:15.502135Z","iopub.status.idle":"2024-10-08T14:50:15.507644Z","shell.execute_reply.started":"2024-10-08T14:50:15.502079Z","shell.execute_reply":"2024-10-08T14:50:15.506672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}