{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.5"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3362,"databundleVersionId":31148,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"ca7a2e43","cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.transforms import v2\nfrom torchvision import models\nfrom torchvision.models.resnet import ResNet, Bottleneck\nimport cv2\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm","metadata":{},"outputs":[],"execution_count":null},{"id":"6751afa4","cell_type":"markdown","source":"# Dataset and DataLoader","metadata":{}},{"id":"2a4ac34d","cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, path, train=True, transform=None):\n        self.path = os.path.join(path, \"train\" if train else \"test\")\n        self.images = os.listdir(self.path)\n        random.shuffle(self.images)\n        self.labels = [int(img[0]) for img in self.images]\n        self.length = len(self.labels)\n        self.target = torch.eye(2)\n        self.transform = transform\n        self.train = train\n    \n    def __getitem__(self, indx):\n        image = cv2.imread(os.path.join(self.path, self.images[indx])) \n        image = torch.from_numpy(image).permute(2, 0, 1) / 255\n        label = self.labels[indx]\n        target = self.target[label]\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, target\n    \n    def __len__(self):\n        return self.length","metadata":{},"outputs":[],"execution_count":null},{"id":"6f13fbdd","cell_type":"code","source":"path = r\"datasets\\dogs_and_cats\"","metadata":{},"outputs":[],"execution_count":null},{"id":"a403cdd2","cell_type":"code","source":"transforms = v2.Compose( [v2.Resize((360, 360)),\n                          v2.RandomCrop((340, 340)),\n                          v2.RandomAffine(degrees=(-10, 10), \n                                          translate=(0.05, 0.05), \n                                          scale=(0.95, 1.05), \n                                          shear=(-3, 3), \n                                          fill=(0)),\n                          v2.RandomPhotometricDistort(), \n                          v2.RandomHorizontalFlip()] )","metadata":{},"outputs":[],"execution_count":null},{"id":"e5ab815b","cell_type":"code","source":"train_dataset = CatsDogsDataset(path, train=True, transform=transforms)\ntest_dataset = CatsDogsDataset(path, train=False, transform=v2.Resize((340, 340)))","metadata":{},"outputs":[],"execution_count":null},{"id":"5c5cbe9b","cell_type":"code","source":"batch_size = 8","metadata":{},"outputs":[],"execution_count":null},{"id":"9d6958b7","cell_type":"code","source":"train_loader = DataLoader(train_dataset, \n                          batch_size=batch_size, \n                          shuffle=True, \n                          drop_last=True)","metadata":{},"outputs":[],"execution_count":null},{"id":"8c2986c3","cell_type":"code","source":"test_loader = DataLoader(test_dataset, \n                         batch_size=batch_size, \n                         shuffle=False, \n                         drop_last=False)","metadata":{},"outputs":[],"execution_count":null},{"id":"295a787b","cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{},"outputs":[],"execution_count":null},{"id":"f6cb85f0","cell_type":"code","source":"device","metadata":{},"outputs":[],"execution_count":null},{"id":"c47440ae","cell_type":"markdown","source":"# Techniques to be used:","metadata":{}},{"id":"ca5fbdfb","cell_type":"markdown","source":"- **Cosine learning rate decay**: the learning rate will decrease in cosine.\n\n- **No bias decay**: Removing bias from regularisation (weight decay).\n\n- **Stochastic depth**: Some of the layers will be randomly disabled as in Dropout.\n\n- **Transfer learning**","metadata":{}},{"id":"4f186e0f","cell_type":"code","source":"net = nn.Sequential(nn.Linear(10, 10))\noptimizer_ = optim.Adam(net.parameters(), lr=0.1)\nscheduler_ = optim.lr_scheduler.CosineAnnealingLR(optimizer_, T_max=20)","metadata":{},"outputs":[],"execution_count":null},{"id":"27fb8b29","cell_type":"code","source":"lr = []\nfor _ in range(20):\n    lr.append(optimizer_.param_groups[0][\"lr\"])\n    optimizer_.step()\n    scheduler_.step()","metadata":{},"outputs":[],"execution_count":null},{"id":"8c140a04","cell_type":"code","source":"plt.title(\"Cosine learning rate decay\")\nplt.xlabel(\"Epochs\");\nplt.ylabel(\"Learning rate\");\nplt.plot(lr);","metadata":{},"outputs":[],"execution_count":null},{"id":"2900910f","cell_type":"markdown","source":"# Bottleneck block using Stochastic depth","metadata":{}},{"id":"7879ad7c","cell_type":"code","source":"class BottleneckSD(Bottleneck):\n    def __init__(self, *args, drop_prob=0.0, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.drop_prob = drop_prob\n\n    def forward(self, x):\n        identity = x\n        \n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        if self.training and torch.rand(1).item() < self.drop_prob:  \n            return identity\n        \n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n        \n        if self.training:\n            out = out / (1 - self.drop_prob)  \n        \n        out += identity\n        out = self.relu(out)\n        return out","metadata":{},"outputs":[],"execution_count":null},{"id":"dec648bf","cell_type":"markdown","source":"# Resnet152 with new Bottleneck blocks","metadata":{}},{"id":"ffce0347","cell_type":"code","source":"class ResNet152SD(ResNet):\n    def __init__(self, drop_prob=0, **kwargs):\n        super().__init__(BottleneckSD, [3, 8, 36, 3], **kwargs)\n        total_blocks = sum([8, 36, 3])\n        block_id = 1\n        for layer in [self.layer2, self.layer3, self.layer4]:  \n            for i in range(len(layer)):\n                current_drop_prob = drop_prob * float(block_id) / total_blocks\n                layer[i].drop_prob = current_drop_prob\n                block_id += 1","metadata":{},"outputs":[],"execution_count":null},{"id":"126e386a","cell_type":"markdown","source":"# Loading the weights of the pre-trained model","metadata":{}},{"id":"62c3c8ee","cell_type":"code","source":"pretrained_model = models.resnet152(weights=models.ResNet152_Weights.DEFAULT)\nmodel = ResNet152SD(drop_prob=0.15)\nmodel.load_state_dict(pretrained_model.state_dict())","metadata":{},"outputs":[],"execution_count":null},{"id":"338a6ece","cell_type":"code","source":"epochs = 5","metadata":{},"outputs":[],"execution_count":null},{"id":"532f7a5a","cell_type":"markdown","source":"# Freezing a part of the layers","metadata":{}},{"id":"49f62d83","cell_type":"code","source":"model.requires_grad_(False);\n\nfor layer in [model.layer2, model.layer3, model.layer4, model.fc]:\n    layer.requires_grad_(True)","metadata":{},"outputs":[],"execution_count":null},{"id":"c80f534f","cell_type":"code","source":"model.fc = nn.Linear(2048, 2).requires_grad_(True)\nmodel = model.to(device)","metadata":{},"outputs":[],"execution_count":null},{"id":"861ca222","cell_type":"markdown","source":"# No bias decay","metadata":{}},{"id":"def91a2e","cell_type":"code","source":"decay = dict()\nno_decay = dict()\nfor name, param in model.named_parameters():\n    if 'weight' in name:\n        decay[name] = param\n    else:\n        no_decay[name] = param","metadata":{},"outputs":[],"execution_count":null},{"id":"01c4def2","cell_type":"code","source":"layer2_weights = []\nlayer3_weights = []\nlayer4_weights = []\nfc_weights = []\nfor name, weight in decay.items():\n    if \"layer2\" in name:\n        layer2_weights.append(weight)\n    elif \"layer3\" in name:\n        layer3_weights.append(weight)\n    elif \"layer4\" in name:\n        layer4_weights.append(weight)\n    elif \"fc\" in name:\n        fc_weights.append(weight)\n    else:\n        print(f\"{name} is not included!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"7c1b08f8","cell_type":"code","source":"layer2_bias = []\nlayer3_bias = []\nlayer4_bias = []\nfc_bias = []\nfor name, weight in no_decay.items():\n    if \"layer2\" in name:\n        layer2_bias.append(weight)\n    elif \"layer3\" in name:\n        layer3_bias.append(weight)\n    elif \"layer4\" in name:\n        layer4_bias.append(weight)\n    elif \"fc\" in name:\n        fc_bias.append(weight)\n    else:\n        print(f\"{name} is not included!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"2626bfaf","cell_type":"markdown","source":"# Each layer will have its own learning rate. The learning rate will be lower for earlier layers, and higher for later layers. We remove regularisation for bias.","metadata":{}},{"id":"ebcaa922","cell_type":"code","source":"optimizer = optim.AdamW([\n    {\"params\": layer2_weights, \"lr\": 10**(-6)},\n    {\"params\": layer3_weights, \"lr\": 10**(-5)},\n    {\"params\": layer4_weights, \"lr\": 10**(-4)},\n    {\"params\": fc_weights, \"lr\": 10**(-3)},\n    {\"params\": layer2_bias, \"lr\": 10**(-6), \"weight_decay\": 0},\n    {\"params\": layer3_bias, \"lr\": 10**(-5), \"weight_decay\": 0},\n    {\"params\": layer4_bias, \"lr\": 10**(-4), \"weight_decay\": 0},\n    {\"params\": fc_bias, \"lr\": 10**(-3), \"weight_decay\": 0}\n], weight_decay=10**(-4))","metadata":{},"outputs":[],"execution_count":null},{"id":"01332c86","cell_type":"markdown","source":"# Cosine learning rate decay","metadata":{}},{"id":"d6d9ac7f","cell_type":"code","source":"scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs*2)\ncriterion = nn.CrossEntropyLoss().to(device)","metadata":{},"outputs":[],"execution_count":null},{"id":"3bb4f1c7","cell_type":"markdown","source":"# Training of the model","metadata":{}},{"id":"68da0533","cell_type":"code","source":"loss_ = []\naccuracy_ = []\n\nfor epoch in range(1, epochs+1):\n    model.train()\n    \n    progress_bar = tqdm(train_loader,\n                        leave=True,\n                        unit=\"batch\")\n    \n    mean_loss = 0\n    m = 0\n    \n    for images, labels in progress_bar:\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        optimizer.zero_grad()\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        \n        optimizer.step()\n        scheduler.step()\n        \n        value = loss.item()\n        m += 1\n        mean_loss = 1/m * value + (1 - 1/m) * mean_loss\n        \n        text = f\"[train] Epoch: [{epoch}/{epochs}] <Loss>: {mean_loss:.2f}|\"\n        progress_bar.set_description(text)\n    \n    model.eval()\n    total = 0\n    correct = 0\n    with torch.no_grad():\n        for images, labels in test_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n\n            outputs = model(images)\n\n            predicted = torch.argmax(outputs, dim=1)\n            labels = torch.argmax(labels, dim=1)\n\n            total += labels.shape[0]\n            correct += (predicted == labels).sum().item()\n\n    accuracy = 100 * correct / total\n    print(f\"Accuracy: {accuracy}\")\n    \n    loss_.append(mean_loss)\n    accuracy_.append(accuracy)","metadata":{},"outputs":[],"execution_count":null},{"id":"120b016d","cell_type":"code","source":"plt.plot(range(1, epochs+1), loss_);\nplt.title(\"Loss\");\nplt.xticks(range(1, epochs+1))\nplt.xlabel(\"Epochs\");","metadata":{},"outputs":[],"execution_count":null},{"id":"25a10d87","cell_type":"code","source":"plt.plot(range(1, epochs+1), accuracy_);\nplt.title(\"Accuracy\");\nplt.xticks(range(1, epochs+1))\nplt.xlabel(\"Epochs\");","metadata":{},"outputs":[],"execution_count":null},{"id":"67c8e2c4","cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}