{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, MultiStepLR\nfrom torch.utils.data import DataLoader\nfrom torchvision import models\nfrom torchvision import transforms\n\n# Define Residual Block\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n        super(ResidualBlock, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        self.downsample = downsample\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n# Define ResNet\nclass ResNet(nn.Module):\n    def __init__(self, block, layers, num_classes=10):\n        super(ResNet, self).__init__()\n        self.in_channels = 16\n        self.conv = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn = nn.BatchNorm2d(16)\n        self.relu = nn.ReLU(inplace=True)\n        self.layer1 = self.make_layer(block, 16, layers[0])\n        self.layer2 = self.make_layer(block, 32, layers[1], 2)\n        self.layer3 = self.make_layer(block, 64, layers[2], 2)\n        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(64, num_classes)\n\n    def make_layer(self, block, out_channels, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.in_channels != out_channels:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels)\n            )\n\n        layers = []\n        layers.append(block(self.in_channels, out_channels, stride, downsample))\n        self.in_channels = out_channels\n        for _ in range(1, blocks):\n            layers.append(block(out_channels, out_channels))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv(x)\n        out = self.bn(out)\n        out = self.relu(out)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.avg_pool(out)\n        out = out.view(out.size(0), -1)\n        out = self.fc(out)\n        return out\n\n# Data Augmentation and Normalization\ntransform_train = transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n])\n\ntransform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n])\n\n# CIFAR-10 Dataset\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)\ntest_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n\n# Define Data Loaders\ntrain_loader = DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=4)\ntest_loader = DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=4)\n\n# Initialize Model, Criterion, Optimizer\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = ResNet(ResidualBlock, [2, 2, 2]).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.0001)\nscheduler = CosineAnnealingLR(optimizer, T_max=200)\n\n# Training Loop\ndef train(epoch):\n    model.train()\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.to(device), target.to(device)\n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        if batch_idx % 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                epoch, batch_idx * len(data), len(train_loader.dataset),\n                100. * batch_idx / len(train_loader), loss.item()))\n\n# Testing Loop\ndef test():\n    model.eval()\n    test_loss = 0\n    correct = 0\n    with torch.no_grad():\n        for data, target in test_loader:\n            data, target = data.to(device), target.to(device)\n            output = model(data)\n            test_loss += criterion(output, target).item()\n            pred = output.argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    test_loss /= len(test_loader.dataset)\n    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\\n'.format(\n        test_loss, correct, len(test_loader.dataset),\n        100. * correct / len(test_loader.dataset)))\n\n# Training and Evaluation\nepochs = 10\nfor epoch in range(1, epochs + 1):\n    train(epoch)\n    test()\n    scheduler.step()\n\n# Save the trained model\ntorch.save(model.state_dict(), \"resnet_cifar10.pth\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T16:28:54.624372Z","iopub.execute_input":"2024-10-12T16:28:54.624778Z","iopub.status.idle":"2024-10-12T16:30:57.776006Z","shell.execute_reply.started":"2024-10-12T16:28:54.624741Z","shell.execute_reply":"2024-10-12T16:30:57.774522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import DataLoader\nimport numpy as np\nimport random\nfrom torchvision import models  # <-- This line is added to fix the error\n\n# Define CutOut augmentation\nclass Cutout(object):\n    def __init__(self, n_holes, length):\n        self.n_holes = n_holes\n        self.length = length\n\n    def __call__(self, img):\n        h, w = img.size(1), img.size(2)\n        mask = np.ones((h, w), np.float32)\n\n        for _ in range(self.n_holes):\n            y = np.random.randint(h)\n            x = np.random.randint(w)\n\n            y1 = np.clip(y - self.length // 2, 0, h)\n            y2 = np.clip(y + self.length // 2, 0, h)\n            x1 = np.clip(x - self.length // 2, 0, w)\n            x2 = np.clip(x + self.length // 2, 0, w)\n\n            mask[y1: y2, x1: x2] = 0.\n\n        mask = torch.from_numpy(mask)\n        mask = mask.expand_as(img)\n        img = img * mask\n        return img\n\n# Deeper ResNet (ResNet-18)\nclass ResNet18(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ResNet18, self).__init__()\n        self.resnet = models.resnet18(weights=None)  # Randomly initialized weights\n        self.resnet.fc = nn.Linear(self.resnet.fc.in_features, num_classes)  # Modify the last layer for CIFAR-10\n\n    def forward(self, x):\n        return self.resnet(x)\n\n# Data Augmentation with CutOut\ntransform_train = transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    Cutout(n_holes=1, length=8),  # CutOut augmentation\n])\n\ntransform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n])\n\n# CIFAR-10 Dataset\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)\ntest_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n\n# Data Loaders\ntrain_loader = DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=4)\ntest_loader = DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=4)\n\n# Initialize Model, Loss, Optimizer, and Scheduler\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = ResNet18().to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.0005)  # AdamW with weight decay\n\n# Cosine Annealing with Warmup\nscheduler = CosineAnnealingLR(optimizer, T_max=200)\n\n# Training Loop\ndef train(epoch):\n    model.train()\n    running_loss = 0.0\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.to(device), target.to(device)\n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n\n        if batch_idx % 100 == 0:\n            print('Train Epoch: {} [{}/{}]\\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset), loss.item()))\n\n    print('Training Loss after epoch {}: {:.4f}'.format(epoch, running_loss / len(train_loader)))\n\n# Testing Loop\ndef test():\n    model.eval()\n    test_loss = 0\n    correct = 0\n    with torch.no_grad():\n        for data, target in test_loader:\n            data, target = data.to(device), target.to(device)\n            output = model(data)\n            test_loss += criterion(output, target).item()  # sum up batch loss\n            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    test_loss /= len(test_loader.dataset)\n    accuracy = 100. * correct / len(test_loader.dataset)\n    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\\n'.format(test_loss, correct, len(test_loader.dataset), accuracy))\n    return accuracy\n\n# Training and Evaluation\nbest_accuracy = 0\nfor epoch in range(1, 101):\n    train(epoch)\n    accuracy = test()\n    scheduler.step()  # Update learning rate based on the scheduler\n    if accuracy > best_accuracy:\n        best_accuracy = accuracy\n        torch.save(model.state_dict(), \"best_resnet18_cifar10.pth\")  # Save best model\n\nprint(f\"Best Accuracy: {best_accuracy:.2f}%\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T16:42:18.868300Z","iopub.execute_input":"2024-10-12T16:42:18.868721Z","iopub.status.idle":"2024-10-12T17:26:25.507597Z","shell.execute_reply.started":"2024-10-12T16:42:18.868679Z","shell.execute_reply":"2024-10-12T17:26:25.506380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-10-12T17:35:50.113264Z","iopub.execute_input":"2024-10-12T17:35:50.113678Z","iopub.status.idle":"2024-10-12T17:35:52.375990Z","shell.execute_reply.started":"2024-10-12T17:35:50.113637Z","shell.execute_reply":"2024-10-12T17:35:52.374732Z"},"trusted":true},"execution_count":null,"outputs":[]}]}