{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":63958,"databundleVersionId":7241949,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2024-01-12T19:30:09.781453Z","iopub.status.idle":"2024-01-12T19:30:09.782680Z","shell.execute_reply.started":"2024-01-12T19:30:09.782344Z","shell.execute_reply":"2024-01-12T19:30:09.782379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport os\nfrom multiprocessing import freeze_support\nfrom typing import List, Union\nimport gc\nfrom time import time\nfrom functools import wraps\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.datasets import CIFAR10\nfrom torchvision.datasets import CIFAR100\nfrom torchvision.transforms import v2\nfrom tqdm import tqdm\nimport random\nimport pandas as pd\nimport numpy as np\nimport wandb\n\n\ndef timed(fn):\n    @wraps(fn)\n    def wrap(*args, **kwargs):\n        gc.collect()\n        start = time()\n        ret = fn(*args, **kwargs)\n        end = time()\n        return ret, end - start\n\n    return wrap\n\n\ndef get_default_device():\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    if torch.backends.mps.is_available():\n        return torch.device('mps')\n    return torch.device('cpu')\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, cfg, stride=1, downsample=None):\n        super(Bottleneck, self).__init__()\n        self.bn1 = nn.BatchNorm2d(inplanes)\n        self.conv1 = nn.Conv2d(cfg[0], cfg[1], kernel_size=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(cfg[1])\n        self.conv2 = nn.Conv2d(cfg[1], cfg[2], kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(cfg[2])\n        self.conv3 = nn.Conv2d(cfg[2], planes * 4, kernel_size=1, bias=False)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.bn1(x)\n        out = self.relu(out)\n        out = self.conv1(out)\n\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n\n        out = self.bn3(out)\n        out = self.relu(out)\n        out = self.conv3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n\n        return out\n\n# Model built by George Stoica. Retrieved from: https://github.com/Tensor-Reloaded/Advanced-Topics-in-Neural-Networks-Template-2023/blob/main/Lab10/CIFAR10/main.py\nclass PreResNet(nn.Module):\n    def __init__(self, out_features, depth=164):\n        super(PreResNet, self).__init__()\n        assert (depth - 2) % 9 == 0, 'depth should be 9n+2'\n\n        n = (depth - 2) // 9\n        block = Bottleneck\n\n        cfg = [[16, 16, 16],\n               [64, 16, 16] * (n - 1),\n               [64, 32, 32],\n               [128, 32, 32] * (n - 1),\n               [128, 64, 64],\n               [256, 64, 64] * (n - 1),\n               [256]]\n        cfg = [item for sub_list in cfg for item in sub_list]\n\n        self.inplanes = 16\n        self.fc = nn.Linear(cfg[-1], out_features)\n        in_channels = 3\n\n        self.conv1 = nn.Conv2d(in_channels, 16, kernel_size=3, padding=1, bias=False)\n        self.layer1 = self._make_layer(block, 16, n, cfg=cfg[0:3 * n])\n        self.layer2 = self._make_layer(block, 32, n, cfg=cfg[3 * n:6 * n], stride=2)\n        self.layer3 = self._make_layer(block, 64, n, cfg=cfg[6 * n:9 * n], stride=2)\n        self.bn = nn.BatchNorm2d(64 * block.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.avgpool = nn.AvgPool2d(8)\n\n        self.init_model()\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(0.5)\n                m.bias.data.zero_()\n\n    def init_model(self):\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.kaiming_normal_(m.weight, mode='fan_in')\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_in')\n                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                # nn.init.normal_(m.weight, 0, math.sqrt(2. / n))\n            if isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                # nn.init.constant_(m.weight, 0.5)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, cfg, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=1, stride=stride, bias=False),\n            )\n\n        layers = [block(self.inplanes, planes, cfg[0:3], stride, downsample)]\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, cfg[3 * i: 3 * (i + 1)]))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n\n        x = self.layer1(x)  # 32x32\n        x = self.layer2(x)  # 16x16\n        x = self.layer3(x)  # 8x8\n        x = self.bn(x)\n        x = self.relu(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n\n\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n        super(ResidualBlock, self).__init__()\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1),\n            nn.BatchNorm2d(out_channels),\n            nn.LeakyReLU(inplace=True))\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(out_channels))\n        self.downsample = downsample\n        self.relu = nn.LeakyReLU(inplace=True)\n        self.out_channels = out_channels\n\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.conv2(out)\n        if self.downsample:\n            residual = self.downsample(x)\n        out += residual\n        out = self.relu(out)\n        return out\n\n\nclass ResNet(nn.Module):\n    def __init__(self, block, layers, num_classes=100):\n        super(ResNet, self).__init__()\n        self.inplanes = 64\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3),\n            nn.BatchNorm2d(64),\n            nn.ReLU())\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512, num_classes)\n        self.init_model()\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(0.5)\n                m.bias.data.zero_()\n\n    def init_model(self):\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.kaiming_normal_(m.weight, mode='fan_in')\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_in')\n            if isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes, kernel_size=1, stride=stride),\n                nn.BatchNorm2d(planes),\n            )\n        layers = [block(self.inplanes, planes, stride, downsample)]\n        self.inplanes = planes\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.maxpool(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n\n\nclass CachedDataset(Dataset):\n    def __init__(self, dataset: Dataset, runtime_transforms: List[Union[v2.Transform, nn.Module]], cache: bool):\n        if cache:\n            dataset = tuple([x for x in dataset])\n        self.dataset = dataset\n        if not len(runtime_transforms):\n            runtime_transforms.append(nn.Identity())\n        self.runtime_transforms = nn.Sequential(*runtime_transforms)\n\n    def __len__(self):\n        return len(self.dataset)\n\n    def __getitem__(self, i):\n        image, label = self.dataset[i]\n        return self.runtime_transforms(image), label\n\n\nclass CachedDetailedDataset(Dataset):\n    def __init__(self, dataset: list, cache: bool):\n        if cache:\n            dataset = tuple([x for x in dataset])\n        self.dataset = dataset\n\n    def __len__(self):\n        return len(self.dataset)\n\n    def __getitem__(self, i):\n        id, image, label = self.dataset[i]\n        return id, image, label\n\n\ndef get_dataset(train: bool):\n    initial_transforms = v2.Compose([\n        v2.ToTensor(),\n        v2.Normalize(\n            mean=(0.5071, 0.4867, 0.4408),\n            std=(0.2675, 0.2565, 0.2761)\n        ),\n    ])\n    cifar100 = CIFAR100(root='/kaggle/input/cifar100-image-classification-uaic-fii-atnn-2023', train=train, transform=initial_transforms, download=False)\n    runtime_transforms = []\n    if train:\n        runtime_transforms = [\n            v2.RandomHorizontalFlip(),\n            v2.RandomVerticalFlip(),\n            v2.RandomErasing()\n        ]\n    return CachedDataset(cifar100, runtime_transforms, True)\n\n\n@torch.jit.script\ndef accuracy(output, labels):\n    fp_plus_fn = torch.logical_not(output == labels).sum().item()\n    all_elements = len(output)\n    return (all_elements - fp_plus_fn) / all_elements\n\n\ndef train(model, train_loader, criterion, optimizer, device):\n    model.train()\n\n    all_outputs = []\n    all_labels = []\n\n    for data, labels in train_loader:\n        data = data.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        output = model(data)\n        loss = criterion(output, labels)\n\n        loss.backward()\n\n        # torch.nn.utils.clip_grad_norm_(model.parameters(), 5)\n\n        optimizer.step()\n        optimizer.zero_grad(set_to_none=True)\n\n        output = output.softmax(dim=1).detach().cpu().squeeze()\n        labels = labels.cpu().squeeze()\n        all_outputs.append(output)\n        all_labels.append(labels)\n\n    all_outputs = torch.cat(all_outputs).argmax(dim=1)\n    all_labels = torch.cat(all_labels)\n\n    return round(accuracy(all_outputs, all_labels), 4)\n\n\ndef val(model, val_loader, device):\n    model.eval()\n\n    all_outputs = []\n    all_labels = []\n\n    for data, labels in val_loader:\n        data = data.to(device, non_blocking=True)\n\n        with torch.no_grad():\n            output = model(data)\n\n        output = output.softmax(dim=1).cpu().squeeze()\n        labels = labels.squeeze()\n        all_outputs.append(output)\n        all_labels.append(labels)\n\n    all_outputs = torch.cat(all_outputs).argmax(dim=1)\n    all_labels = torch.cat(all_labels)\n\n    return round(accuracy(all_outputs, all_labels), 4)\n\n\ndef train_model(model, train_loader, criterion, optimizer, device, mapping):\n    model.train()\n\n    all_outputs = []\n    all_labels = []\n\n    for tuples in train_loader:\n        if len(tuples) == 2:\n            data, labels = tuples\n        else:\n            id, data, labels = tuples\n        coarse_labels = labels.detach().clone()\n\n        for i in range(len(coarse_labels)):\n            try:\n                coarse_labels[i] = mapping[coarse_labels[i].item()]\n            except KeyError:\n                # TODO: uncomment first line and comment the second one in order to train without added noise\n                # coarse_labels[i] = -1\n                coarse_labels[i] = random.randint(0, len(mapping) - 1)\n\n        data = data.to(device, non_blocking=True)\n        coarse_labels = coarse_labels.to(device, non_blocking=True)\n\n        output = model(data)\n        # TODO: uncomment in order to train without added noise\n        # n = len(coarse_labels)\n        # i = 0\n        # while i < n:\n        #     if coarse_labels[i] == -1:\n        #         output = torch.cat([output[0:i], output[i + 1:]])\n        #         coarse_labels = torch.cat([coarse_labels[0:i], coarse_labels[i + 1:]])\n        #         i -= 1\n        #         n -= 1\n        #     i += 1\n\n        loss = criterion(output, coarse_labels)\n\n        loss.backward()\n\n        optimizer.step()\n        optimizer.zero_grad(set_to_none=True)\n\n        output = output.softmax(dim=1).detach().cpu().squeeze()\n        coarse_labels = coarse_labels.cpu().squeeze()\n        all_outputs.append(output)\n        all_labels.append(coarse_labels)\n\n    all_outputs = torch.cat(all_outputs)\n    all_outputs_top1 = all_outputs.argmax(dim=1)\n    all_outputs_top3 = torch.topk(all_outputs, k=3, dim=1).indices\n    all_labels = torch.cat(all_labels)\n\n    return round(accuracy(all_outputs_top1, all_labels), 4), round(top3_accuracy(all_outputs_top3, all_labels), 4)\n\n\ndef val_model(model, val_loader, device, mapping):\n    model.eval()\n\n    all_outputs = []\n    all_labels = []\n\n    for tuples in val_loader:\n        if len(tuples) == 2:\n            data, labels = tuples\n        else:\n            id, data, labels = tuples\n        coarse_labels = labels.detach().clone()\n        for i in range(len(coarse_labels)):\n            try:\n                coarse_labels[i] = mapping[coarse_labels[i].item()]\n            except KeyError:\n                coarse_labels[i] = -1\n\n        data = data.to(device, non_blocking=True)\n        coarse_labels = coarse_labels.to(device, non_blocking=True)\n\n        with torch.no_grad():\n            output = model(data)\n\n        n = len(coarse_labels)\n        i = 0\n        while i < n:\n            if coarse_labels[i] == -1:\n                output = torch.cat([output[0:i], output[i + 1:]])\n                coarse_labels = torch.cat([coarse_labels[0:i], coarse_labels[i + 1:]])\n                i -= 1\n                n -= 1\n            i += 1\n\n        output = output.softmax(dim=1).cpu().squeeze()\n        coarse_labels = coarse_labels.cpu().squeeze()\n        all_outputs.append(output)\n        all_labels.append(coarse_labels)\n\n    all_outputs = torch.cat(all_outputs)\n    all_outputs_top1 = all_outputs.argmax(dim=1)\n    all_outputs_top3 = torch.topk(all_outputs, k=3, dim=1).indices\n    all_labels = torch.cat(all_labels)\n\n    return round(accuracy(all_outputs_top1, all_labels), 4), round(top3_accuracy(all_outputs_top3, all_labels), 4)\n\n\ndef do_epoch_for_coarse_model(model, train_loader, val_loader, criterion, optimizer, device, mapping):\n    acc, top3_acc = train_model(model, train_loader, criterion, optimizer, device, mapping)\n    acc_val, top3_acc_val = val_model(model, val_loader, device, mapping)\n    return acc, acc_val\n\n\ndef do_epoch_for_detailed_models(first_model, second_model, third_model, fourth_model,\n                                 first_train_loader, second_train_loader, third_train_loader, fourth_train_loader,\n                                 first_test_loader, second_test_loader, third_test_loader, fourth_test_loader,\n                                 criterion_first_category, optimizer_first_category, criterion_second_category,\n                                 optimizer_second_category, criterion_third_category, optimizer_third_category,\n                                 criterion_fourth_category, optimizer_fourth_category, device,\n                                 first_mapping, second_mapping, third_mapping, fourth_mapping):\n    acc_first_detailed_model, top3_acc_first = train_model(first_model, first_train_loader, criterion_first_category,\n                                                           optimizer_first_category, device, first_mapping)\n    acc_first_detailed_model_val, top3_acc_first_val = val_model(first_model, first_test_loader, device, first_mapping)\n\n    acc_second_detailed_model, top3_acc_second = train_model(second_model, second_train_loader,\n                                                             criterion_second_category,\n                                                             optimizer_second_category, device, second_mapping)\n    acc_second_detailed_model_val, top3_acc_second_val = val_model(second_model, second_test_loader, device,\n                                                                   second_mapping)\n\n    acc_third_detailed_model, top3_acc_third = train_model(third_model, third_train_loader, criterion_third_category,\n                                                           optimizer_third_category, device, third_mapping)\n    acc_third_detailed_model_val, top3_acc_third_val = val_model(third_model, third_test_loader, device, third_mapping)\n\n    acc_fourth_detailed_model, top3_acc_fourth = train_model(fourth_model, fourth_train_loader, criterion_fourth_category,\n                                                           optimizer_fourth_category, device, fourth_mapping)\n    acc_fourth_detailed_model_val, top3_acc_fourth_val = val_model(fourth_model, fourth_test_loader, device, fourth_mapping)\n\n    return (acc_first_detailed_model, acc_second_detailed_model, acc_third_detailed_model, acc_fourth_detailed_model,\n            round((acc_first_detailed_model + acc_second_detailed_model + acc_third_detailed_model + acc_fourth_detailed_model) / 4, 4),\n            round((acc_first_detailed_model_val + acc_second_detailed_model_val + acc_third_detailed_model_val + acc_fourth_detailed_model_val) / 4, 4),\n            round((top3_acc_first + top3_acc_second + top3_acc_third + top3_acc_fourth) / 4, 4),\n            round((top3_acc_first_val + top3_acc_second_val + top3_acc_third_val + top3_acc_fourth_val) / 4, 4))\n\n\ndef get_detailed_train_datasets(model, train_loader, device):\n    first_category_data = []\n    second_category_data = []\n    third_category_data = []\n    fourth_category_data = []\n    start_pos = 0\n    for data, labels in train_loader:\n        data = data.to(device, non_blocking=True)\n\n        with torch.no_grad():\n            output = model(data)\n        output = output.softmax(dim=1).detach().cpu().squeeze().argmax(dim=1)\n\n        for i in range(len(output)):\n            match output[i].item():\n                case 0:\n                    first_category_data.append((start_pos, data[i], labels[i].item()))\n                case 1:\n                    second_category_data.append((start_pos, data[i], labels[i].item()))\n                case 2:\n                    third_category_data.append((start_pos, data[i], labels[i].item()))\n                case 3:\n                    fourth_category_data.append((start_pos, data[i], labels[i].item()))\n            start_pos += 1\n\n    first_category_dataset = CachedDetailedDataset(first_category_data, True)\n    second_category_dataset = CachedDetailedDataset(second_category_data, True)\n    third_category_dataset = CachedDetailedDataset(third_category_data, True)\n    fourth_category_dataset = CachedDetailedDataset(fourth_category_data, True)\n\n    return first_category_dataset, second_category_dataset, third_category_dataset, fourth_category_dataset\n\n\ndef get_detailed_test_datasets(model, test_loader, device):\n    first_category_data = []\n    second_category_data = []\n    third_category_data = []\n    fourth_category_data = []\n    start_pos = 0\n    # TODO: have to modify this into two methods, one for train, and another one for test because i don't have labels on kaggle\n    for data, labels in test_loader:\n        data = data.to(device, non_blocking=True)\n\n        with torch.no_grad():\n            output = model(data)\n        output = output.softmax(dim=1).detach().cpu().squeeze().argmax(dim=1)\n\n        for i in range(len(output)):\n            match output[i].item():\n                case 0:\n                    first_category_data.append((start_pos, data[i], labels[i].item()))\n                case 1:\n                    second_category_data.append((start_pos, data[i], labels[i].item()))\n                case 2:\n                    third_category_data.append((start_pos, data[i], labels[i].item()))\n                case 3:\n                    fourth_category_data.append((start_pos, data[i], labels[i].item()))\n            start_pos += 1\n\n    first_category_dataset = CachedDetailedDataset(first_category_data, True)\n    second_category_dataset = CachedDetailedDataset(second_category_data, True)\n    third_category_dataset = CachedDetailedDataset(third_category_data, True)\n    fourth_category_dataset = CachedDetailedDataset(fourth_category_data, True)\n\n    return first_category_dataset, second_category_dataset, third_category_dataset, fourth_category_dataset\n\n\ndef create_components(config, device, out_features):\n    train_dataset = get_dataset(train=True)\n    val_dataset = get_dataset(train=False)\n\n    # model = PreResNet(out_features, 56)\n    model = ResNet(ResidualBlock, [2, 2, 2, 2], out_features)\n    model = model.to(device)\n    if config[\"optimizer\"] == \"RMSProp\":\n        optimizer = torch.optim.RMSprop(model.parameters(), config[\"learning_rate\"], weight_decay=config[\"decay\"],\n                                        momentum=config[\"momentum\"])\n    elif config[\"optimizer\"] == \"Adam\":\n        optimizer = torch.optim.Adam(model.parameters(), config[\"learning_rate\"], weight_decay=config[\"decay\"])\n    elif config[\"optimizer\"] == \"SGD\":\n        optimizer = torch.optim.SGD(model.parameters(), config[\"learning_rate\"], weight_decay=config[\"decay\"],\n                                    momentum=config[\"momentum\"], nesterov=True)\n    elif config[\"optimizer\"] == \"AdaGrad\":\n        optimizer = torch.optim.Adagrad(model.parameters(), config[\"learning_rate\"], weight_decay=config[\"decay\"])\n    else:\n        optimizer = torch.optim.SGD(model.parameters(), config[\"learning_rate\"])\n\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.2, patience=10,\n                                                           threshold=0.001, threshold_mode='rel')\n    criterion = torch.nn.CrossEntropyLoss()\n    num_workers = 0\n    persistent_workers = (num_workers != 0) and False\n    train_loader = DataLoader(train_dataset, shuffle=True, pin_memory=(device.type == 'cuda'), num_workers=num_workers,\n                              batch_size=config[\"batch_size_test1\"], drop_last=True,\n                              persistent_workers=persistent_workers)\n    val_loader = DataLoader(val_dataset, shuffle=False, pin_memory=True, num_workers=0,\n                            batch_size=config[\"batch_size_val1\"],\n                            drop_last=False)\n\n    return model, train_loader, val_loader, criterion, optimizer, scheduler\n\n\ndef top3_accuracy(output, labels):\n    element_exists = (labels[:, None] == output)\n    num_rows_with_condition = torch.sum(element_exists, dim=1).sum().item()\n    all_elements = len(labels)\n    return num_rows_with_condition / all_elements\n\n\ndef predict_model(model, val_loader, device, mapping, results_dict):\n    model.eval()\n\n    for id, data, labels in val_loader:\n\n        data = data.to(device, non_blocking=True)\n\n        with torch.no_grad():\n            output = model(data)\n\n        output = output.softmax(dim=1).cpu().squeeze()\n        for i in range(len(output)):\n            final_result = np.zeros(100)\n            probabilities = np.round(output[i].numpy(), 4)\n            for j in range(len(probabilities)):\n                final_result[mapping[j]] = probabilities[j]\n            results_dict[id[i].item()] = final_result.tolist()\n\n\ndef create_dataframe(results_dict):\n    data = {\n        \"ID\": [],\n        \"target\": []\n    }\n    for i in range(len(results_dict)):\n        data[\"ID\"].append(i)\n        data[\"target\"].append(results_dict[i])\n    return data\n\n\ndef predict(first_model, second_model, third_model, fourth_model, first_test_loader, second_test_loader,\n            third_test_loader, fourth_test_loader, device, first_reversed_mapping, second_reversed_mapping,\n            third_reversed_mapping, fourth_reversed_mapping):\n    # TODO: de completat partea asta pt fiecare model, si de facut mai sus functia care sa faca solutia pt kaggle\n    results_dict = dict()\n    predict_model(first_model, first_test_loader, device, first_reversed_mapping, results_dict)\n    predict_model(second_model, second_test_loader, device, second_reversed_mapping, results_dict)\n    predict_model(third_model, third_test_loader, device, third_reversed_mapping, results_dict)\n    predict_model(fourth_model, fourth_test_loader, device, fourth_reversed_mapping, results_dict)\n    submission_data = create_dataframe(results_dict)\n    df = pd.DataFrame(submission_data)\n    df.to_csv(\"/kaggle/working/first_submission.csv\", index=False)\n\n\ndef main(hyperparameters, device: torch.device = get_default_device(), data_path: str = './data',\n         models_path: str = \"./models\"):\n    os.makedirs(models_path, exist_ok=True)\n    if device.type == 'cuda':\n        torch.backends.cudnn.benchmark = True\n        torch.backends.cuda.matmul.allow_tf32 = True\n        \n    model, train_loader, val_loader, criterion, optimizer, scheduler = create_components(hyperparameters, device, 4)\n\n    # First class (27 elements): apple(0), bee(6), bottle(9), bowl(10), butterfly(14), can(16), caterpillar(18),\n    #                           clock(22), cockroach(24) cup(28), keyboard(39), lamp(40),\n    #                           mushroom(51), orange(53), orchid(54), pear(57), plate(61), poppy(62), rose(70), snail(77), snake(78),\n    #                           spider(79), sunflower(82), sweet_pepper(83), telephone(86), tulip(92), worm(99)\n    # Second class (29 elements): baby(2), bear(3), beaver(4), beetle(7), boy(11),\n    #                           camel(15), cattle(19), chimpanzee(21),\n    #                          dinosaur(29), elephant(31),\n    #                         fox(34), girl(35), hamster(36), kangaroo(38), leopard(42), lion(43), lizard(44),\n    #                         man(46), mouse(50), porcupine(63), possum(64), rabbit(65),\n    #                         raccoon(66), shrew(74), skunk(75),\n    #                         squirrel(80), tiger(88), wolf(97),\n    #                         woman(98)\n    # Third class (31 elements): bed(5), bicycle(8), bridge(12), bus(13), castle(17), chair(20), cloud(23),\n    #                           couch(25), forest(33), house(37), lawn_mower(41), maple_tree(47), motorcycle(48),\n    #                           mountain(49), oak_tree(52), palm_tree(56), pickup_truck(58), pine_tree(59),\n    #                           plain(60), road(68), rocket(69), sea(71), skyscraper(76), streetcar(81), table(84),\n    #                           tank(85), television(87), tractor(89), train(90), wardrobe(94), willow_tree(96)\n    # Forth class (13 elements): aquarium_fish(1), crab(26), crocodile(27), dolphin(30), flatfish(32), lobster(45), otter(55),\n    #                            ray(67), seal(72), shark(73), trout(91), turtle(93), whale(95),\n    coarse_mapping = {0: 0, 6: 0, 9: 0, 10: 0, 14: 0, 16: 0, 18: 0, 22: 0, 24: 0, 28: 0, 39: 0, 40: 0, 51: 0, 53: 0, 54: 0, 57: 0, 61: 0,\n                      62: 0, 70: 0, 77: 0, 78: 0, 79: 0, 82: 0, 83: 0, 86: 0, 92: 0, 99: 0,\n                      2: 1, 3: 1, 4: 1, 7: 1, 11: 1, 15: 1, 19: 1, 21: 1,\n                      29: 1, 31: 1, 34: 1, 35: 1, 36: 1, 38: 1, 42: 1, 43: 1, 44: 1,\n                      46: 1, 50: 1, 63: 1, 64: 1, 65: 1, 66: 1, 74: 1, 75: 1,\n                      80: 1, 88: 1, 97: 1, 98: 1,\n                      5: 2, 8: 2, 12: 2, 13: 2, 17: 2, 20: 2, 23: 2, 25: 2, 33: 2, 37: 2, 41: 2, 47: 2, 48: 2,\n                      49: 2, 52: 2, 56: 2, 58: 2, 59: 2, 60: 2, 68: 2, 69: 2, 71: 2, 76: 2, 81: 2, 84: 2, 85: 2,\n                      87: 2, 89: 2, 90: 2, 94: 2, 96: 2, \n                      1: 3, 26: 3, 27: 3,  30: 3, 32: 3, 45: 3, 55: 3, 67: 3,\n                      72: 3, 73: 3, 91: 3, 93: 3, 95: 3,\n                      }\n    # with wandb.init(\n    #         project=\"CIFAR100 Project V2 test \" + hyperparameters[\"optimizer\"],\n    #         config=hyperparameters):\n\n    tbar = tqdm(tuple(range(hyperparameters[\"epochs1\"])))\n    best_val = 0.0\n    for epoch in tbar:\n        acc, acc_val = do_epoch_for_coarse_model(model, train_loader, val_loader, criterion, optimizer, device,\n                                                 coarse_mapping)\n        scheduler.step(acc)\n\n        if acc_val > best_val:\n            #torch.save(model.state_dict(), os.path.join(models_path, \"best_resnet18.pth\"))\n            best_val = acc_val\n        tbar.set_postfix_str(f\"Coarse Training - Acc: {acc}, Acc_val: {acc_val}, Best_val: {best_val}\")\n        #wandb.log({\"epoch\": epoch, \"acc\": acc, \"acc_val\": acc_val})\n    # TODO: de decomentat ce e mai sus si sters ce e aici cu load\n    # model = ResNet(ResidualBlock, [3, 4, 6, 3], 3)\n    # model = model.to(device)\n    # model.load_state_dict(torch.load(os.path.join(models_path, \"best.pth\"), map_location=device))\n\n    # Getting the fine-grained datasets based on the model trained above\n    first_category_train_dataset, second_category_train_dataset, third_category_train_dataset, fourth_category_train_dataset = get_detailed_train_datasets(\n        model, train_loader, device)\n    first_category_test_dataset, second_category_test_dataset, third_category_test_dataset, fourth_category_test_dataset = get_detailed_test_datasets(\n        model, val_loader, device)\n\n    batch_size = hyperparameters[\"batch_size_test2\"]\n    val_batch_size = hyperparameters[\"batch_size_val2\"]\n    num_workers = 0\n    persistent_workers = (num_workers != 0) and False\n\n    first_category_train_loader = DataLoader(first_category_train_dataset, shuffle=True,\n                                             pin_memory=False,\n                                             num_workers=num_workers,\n                                             batch_size=batch_size, drop_last=False,\n                                             persistent_workers=persistent_workers)\n    second_category_train_loader = DataLoader(second_category_train_dataset, shuffle=True,\n                                              pin_memory=False,\n                                              num_workers=num_workers,\n                                              batch_size=batch_size, drop_last=False,\n                                              persistent_workers=persistent_workers)\n    third_category_train_loader = DataLoader(third_category_train_dataset, shuffle=True,\n                                             pin_memory=False,\n                                             num_workers=num_workers,\n                                             batch_size=batch_size, drop_last=False,\n                                             persistent_workers=persistent_workers)\n    fourth_category_train_loader = DataLoader(fourth_category_train_dataset, shuffle=True,\n                                              pin_memory=False,\n                                              num_workers=num_workers,\n                                              batch_size=batch_size, drop_last=False,\n                                              persistent_workers=persistent_workers)\n\n    first_category_test_loader = DataLoader(first_category_test_dataset, shuffle=False, pin_memory=False, num_workers=0,\n                                            batch_size=val_batch_size,\n                                            drop_last=False)\n    second_category_test_loader = DataLoader(second_category_test_dataset, shuffle=False, pin_memory=False,\n                                             num_workers=0,\n                                             batch_size=val_batch_size,\n                                             drop_last=False)\n    third_category_test_loader = DataLoader(third_category_test_dataset, shuffle=False, pin_memory=False, num_workers=0,\n                                            batch_size=val_batch_size,\n                                            drop_last=False)\n    fourth_category_test_loader = DataLoader(fourth_category_test_dataset, shuffle=False, pin_memory=False, num_workers=0,\n                                            batch_size=val_batch_size,\n                                            drop_last=False)\n\n    # First class (27 elements): apple(0), bottle(9), bowl(10), can(16), clock(22), cup(28), keyboard(39), lamp(40),\n    #                           mushroom, orange(53), orchid(54), pear(57), plate(61), poppy(62), rose(70),\n    #                           sunflower(82), sweet_pepper(83), telephone(86), tulip(92), worm(99)\n\n    first_category_mapping = {0: 0, 6: 1, 9: 2, 10: 3, 14: 4, 16: 5, 18: 6, 22: 7, 24: 8, 28: 9, 39: 10, 40: 11, 51: 12, 53: 13, 54: 14, 57: 15,\n                              61: 16, 62: 17, 70: 18, 77: 19, 78: 20, 79: 21, 82: 22, 83: 23, 86: 24, 92: 25, 99: 26}\n\n    # Second class (29 elements): aquarium_fish(1), baby(2), bear(3), beaver(4), bee(6), beetle(7), boy(11),\n    #                         butterfly(14), camel(15), caterpillar(18), cattle(19), chimpanzee(21), cockroach(24),\n    #                         crab(26), crocodile(27), dinosaur(29), dolphin(30), elephant(31), flatfish(32),\n    #                         fox(34), girl(35), hamster(36), kangaroo(38), leopard(42), lion(43), lizard(44),\n    #                         lobster(45), man(46), mouse(50), otter(55), porcupine(63), possum(64), rabbit(65),\n    #                         raccoon(66), ray(67), seal(72), shark(73), shrew(74), skunk(75), snail(77), snake(78),\n    #                         spider(79), squirrel(80), tiger(88), trout(91), whale(95), wolf(97),\n    #                         woman(98)\n\n    second_category_mapping = {2: 0, 3: 1, 4: 2, 7: 3, 11: 4, 15: 5, 19: 6, 21: 7,\n                      29: 8, 31: 9, 34: 10, 35: 11, 36: 12, 38: 13, 42: 14, 43: 15, 44: 16,\n                      46: 17, 50: 18, 63: 19, 64: 20, 65: 21, 66: 22, 74: 23, 75: 24,\n                      80: 25, 88: 26, 97: 27, 98: 28}\n\n    # Third class (31 elements): bed(5), bicycle(8), bridge(12), bus(13), castle(17), chair(20), cloud(23),\n    #                           couch(25), forest(33), house(37), lawn_mower(41), maple_tree(47), motorcycle(48),\n    #                           mountain(49), oak_tree(52), palm_tree(56), pickup_truck(58), pine_tree(59),\n    #                           plain(60), road(68), rocket(69), sea(71), skyscraper(76), streetcar(81), table(84),\n    #                           tank(85), television(87), tractor(89), train(90), wardrobe(94), willow_tree(96)\n    third_category_mapping = {5: 0, 8: 1, 12: 2, 13: 3, 17: 4, 20: 5, 23: 6, 25: 7, 33: 8, 37: 9, 41: 10, 47: 11,\n                              48: 12, 49: 13, 52: 14, 56: 15, 58: 16, 59: 17, 60: 18, 68: 19, 69: 20, 71: 21, 76: 22,\n                              81: 23, 84: 24, 85: 25, 87: 26, 89: 27, 90: 28, 94: 29, 96: 30}\n\n    # Forth class (13 elements): aquarium_fish(1), crab(26), crocodile(27), dolphin(30), flatfish(32), lobster(45), otter(55),\n    #                            ray(67), seal(72), shark(73), trout(91), turtle(93), whale(95),\n    fourth_category_mapping = {1: 0, 26: 1, 27: 2, 30: 3, 32: 4, 45: 5, 55: 6, 67: 7,\n                               72: 8, 73: 9, 91: 10, 93: 11, 95: 12}\n\n    first_category_model = PreResNet(27, 56)\n    #first_category_model = ResNet(ResidualBlock, [2, 2, 2, 2], 27)\n    first_category_model = first_category_model.to(device)\n    optimizer_first_category = torch.optim.SGD(first_category_model.parameters(), lr=0.005, momentum=0.9, nesterov=True,\n                                               weight_decay=0.0001)\n    scheduler_first_category = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_first_category, mode='max',\n                                                                          factor=0.2, patience=10,\n                                                                          threshold=0.001, threshold_mode='rel')\n    criterion_first_category = torch.nn.CrossEntropyLoss()\n\n    second_category_model = PreResNet(29, 56)\n    # second_category_model = ResNet(ResidualBlock, [2, 2, 2, 2], 29)\n    second_category_model = second_category_model.to(device)\n    optimizer_second_category = torch.optim.SGD(second_category_model.parameters(), lr=0.005, momentum=0.9,\n                                                nesterov=True,\n                                                weight_decay=0.0001)\n    scheduler_second_category = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_second_category, mode='max',\n                                                                           factor=0.2, patience=10,\n                                                                           threshold=0.001, threshold_mode='rel')\n    criterion_second_category = torch.nn.CrossEntropyLoss()\n\n    third_category_model = PreResNet(31, 56)\n    # third_category_model = ResNet(ResidualBlock, [2, 2, 2, 2], 31)\n    third_category_model = third_category_model.to(device)\n    optimizer_third_category = torch.optim.SGD(third_category_model.parameters(), lr=0.005, momentum=0.9,\n                                               nesterov=True,\n                                               weight_decay=0.0001)\n    scheduler_third_category = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_third_category, mode='max',\n                                                                          factor=0.2, patience=10,\n                                                                          threshold=0.001, threshold_mode='rel')\n    criterion_third_category = torch.nn.CrossEntropyLoss()\n\n    fourth_category_model = PreResNet(13, 56)\n    # fourth_category_model = ResNet(ResidualBlock, [2, 2, 2, 2], 13)\n    fourth_category_model = fourth_category_model.to(device)\n    optimizer_fourth_category = torch.optim.SGD(fourth_category_model.parameters(), lr=0.005, momentum=0.9,\n                                               nesterov=True,\n                                               weight_decay=0.0001)\n    scheduler_fourth_category = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_fourth_category, mode='max',\n                                                                          factor=0.2, patience=10,\n                                                                          threshold=0.001, threshold_mode='rel')\n    criterion_fourth_category = torch.nn.CrossEntropyLoss()\n\n    tbar = tqdm(tuple(range(hyperparameters[\"epochs2\"])))\n    best_val_top1 = 0.0\n    best_val_top3 = 0.0\n    for epoch in tbar:\n        acc_first, acc_second, acc_third, acc_fourth, acc_top1, acc_val_top1, acc_top3, acc_val_top3 = do_epoch_for_detailed_models(\n            first_category_model,\n            second_category_model,\n            third_category_model,\n            fourth_category_model,\n\n            first_category_train_loader,\n            second_category_train_loader,\n            third_category_train_loader,\n            fourth_category_train_loader,\n\n            first_category_test_loader,\n            second_category_test_loader,\n            third_category_test_loader,\n            fourth_category_test_loader,\n\n            criterion_first_category,\n            optimizer_first_category,\n            criterion_second_category,\n            optimizer_second_category,\n            criterion_third_category,\n            optimizer_third_category,\n            criterion_fourth_category,\n            optimizer_fourth_category,\n            device,\n            first_category_mapping,\n            second_category_mapping,\n            third_category_mapping,\n            fourth_category_mapping)\n        scheduler_first_category.step(acc_first)\n        scheduler_second_category.step(acc_second)\n        scheduler_third_category.step(acc_third)\n        scheduler_fourth_category.step(acc_fourth)\n        if acc_val_top1 > best_val_top1:\n            best_val_top1 = acc_val_top1\n        if acc_val_top3 > best_val_top3:\n            best_val_top3 = acc_val_top3\n        tbar.set_postfix_str(\n            f\"Detailed Training - Acc_top1: {acc_top1}, Acc_val_top1: {acc_val_top1}, Best_val_top1: {best_val_top1}, Acc_top3: {acc_top3}, Acc_val_top3: {acc_val_top3}, Best_val_top3: {best_val_top3}\")\n\n    reversed_first_category_mapping = {0: 0, 1: 6, 2: 9, 3: 10, 4: 14, 5: 16, 6: 18, 7: 22, 8: 24, 9: 28, 10: 39,\n                                       11: 40, 12: 51, 13: 53, 14: 54, 15: 57, 16: 61, 17: 62, 18: 70, 19: 77, 20: 78,\n                                       21: 79, 22: 82, 23: 83, 24: 86, 25: 92, 26: 99}\n\n    reversed_second_category_mapping = {0: 2, 1: 3, 2: 4, 3: 7, 4: 11, 5: 15, 6: 19, 7: 21, 8: 29, 9: 31, 10: 34, 11: 35,\n                                        12: 36, 13: 38, 14: 42, 15: 43, 16: 44, 17: 46, 18: 50, 19: 63, 20: 64, 21: 65,\n                                        22: 66, 23: 74, 24: 75, 25: 80, 26: 88, 27: 97, 28: 98}\n\n    reversed_third_category_mapping = {0: 5, 1: 8, 2: 12, 3: 13, 4: 17, 5: 20, 6: 23, 7: 25, 8: 33, 9: 37, 10: 41,\n                                       11: 47,\n                                       12: 48, 13: 49, 14: 52, 15: 56, 16: 58, 17: 59, 18: 60, 19: 68, 20: 69, 21: 71,\n                                       22: 76,\n                                       23: 81, 24: 84, 25: 85, 26: 87, 27: 89, 28: 90, 29: 94, 30: 96}\n\n    reversed_fourth_category_mapping = {0: 1, 1: 26, 2: 27, 3: 30, 4: 32, 5: 45, 6: 55, 7: 67, 8: 72, 9: 73, 10: 91,\n                                        11: 93, 12: 95}\n\n    predict(first_category_model, second_category_model, third_category_model, fourth_category_model,\n            first_category_test_loader, second_category_test_loader, third_category_test_loader,\n            fourth_category_test_loader, device, reversed_first_category_mapping, reversed_second_category_mapping,\n            reversed_third_category_mapping, reversed_fourth_category_mapping)    \n    \n    \nfreeze_support()\nSGD = dict(\n    epochs1=55,\n    epochs2=70,\n    batch_size_test1=50,\n    batch_size_val1=500,\n    batch_size_test2=30,\n    batch_size_val2=200,\n    learning_rate=0.005,\n    momentum=0.9,\n    decay=0.0005,\n    optimizer=\"SGD\",\n)\nmain(SGD)\n    ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-15T23:31:16.822790Z","iopub.execute_input":"2024-01-15T23:31:16.823140Z","iopub.status.idle":"2024-01-16T00:38:31.434900Z","shell.execute_reply.started":"2024-01-15T23:31:16.823114Z","shell.execute_reply":"2024-01-16T00:38:31.433875Z"},"trusted":true},"execution_count":null,"outputs":[]}]}