{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.datasets as datasets\nfrom torchvision.models import resnet50\nfrom torchvision import transforms\nimport math\n\nbatch_size = 128\nlearning_rate = 1e-3  # Increased learning rate for better optimization\nnum_epochs = 100\nimg_size = 128\n\n# Set seed\nimport os\nimport random\nimport numpy as np\nimport torch\n\nrandom_seed = 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)\n\n# Cutout class for data augmentation\nclass Cutout(object):\n    def __init__(self, n_holes=1, length=16):\n        self.n_holes = n_holes\n        self.length = length\n\n    def __call__(self, img):\n        h = img.size(1)\n        w = img.size(2)\n\n        mask = np.ones((h, w), np.float32)\n\n        for n in range(self.n_holes):\n            y = np.random.randint(h)\n            x = np.random.randint(w)\n\n            y1 = int(np.clip(y - self.length / 2, 0, h))\n            y2 = int(np.clip(y + self.length / 2, 0, h))\n            x1 = int(np.clip(x - self.length / 2, 0, w))\n            x2 = int(np.clip(x + self.length / 2, 0, w))\n\n            mask[y1:y2, x1:x2] = 0.\n\n        mask = torch.from_numpy(mask).expand_as(img)\n        img = img * mask\n\n        return img\n\n# Define transforms\ntrain_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.RandomRotation(20),\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n        Cutout(n_holes=1, length=16),  # Advanced data augmentation\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\ntest_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\n\n# Load data\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)  # Shuffle set to True\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n\n# Squeeze-and-Excitation Block for attention mechanism\nclass SEBlock(nn.Module):\n    def __init__(self, in_channels, reduction=16):\n        super(SEBlock, self).__init__()\n        self.fc1 = nn.Conv2d(in_channels, in_channels // reduction, kernel_size=1)\n        self.fc2 = nn.Conv2d(in_channels // reduction, in_channels, kernel_size=1)\n\n    def forward(self, x):\n        # Squeeze operation\n        out = torch.mean(x, dim=[2, 3], keepdim=True)\n        # Excitation operation\n        out = torch.relu(self.fc1(out))\n        out = torch.sigmoid(self.fc2(out))\n        return x * out\n\n# Define the custom complex model based on ResNet50\nclass ComplexResNet50(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ComplexResNet50, self).__init__()\n        \n        # Initial convolutional block before ResNet50 for richer feature extraction\n        self.pre_conv = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=3, padding=1),  # Output 64 channels, as expected by ResNet50\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Load ResNet50 without pre-trained weights\n        self.resnet50 = resnet50(weights=None)\n        \n        # Skip the initial conv, bn1, relu, and maxpool layers from ResNet50\n        self.resnet50 = nn.Sequential(\n            self.resnet50.layer1,  # Start from layer1 (expects 64 channels)\n            self.resnet50.layer2,\n            self.resnet50.layer3,\n            self.resnet50.layer4\n        )\n        \n        # Add an SE block for attention\n        self.se_block = SEBlock(2048)\n        \n        # Additional convolutional layers after ResNet50\n        self.conv_extra = nn.Sequential(\n            nn.Conv2d(2048, 1024, kernel_size=3, padding=1),\n            nn.BatchNorm2d(1024),\n            nn.ReLU(),\n            nn.Conv2d(1024, 512, kernel_size=3, padding=1),\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Use AdaptiveAvgPool2d to dynamically handle varying input sizes\n        self.adaptive_pool = nn.AdaptiveAvgPool2d((1, 1))  # This will output (512, 1, 1)\n        \n        # Fully connected layers for classification\n        self.fc_layers = nn.Sequential(\n            nn.Linear(512, 1024),  # Adjusted size based on the adaptive pooling output\n            nn.ReLU(),\n            nn.Dropout(0.5),  # Dropout for regularization\n            nn.Linear(1024, 512),\n            nn.ReLU(),\n            nn.BatchNorm1d(512),  # Batch normalization for stability\n            nn.Linear(512, num_classes)\n        )\n\n    def forward(self, x):\n        # Pass through the initial convolutional block\n        x = self.pre_conv(x)\n        \n        # Pass through ResNet50 backbone (starting from layer1)\n        x = self.resnet50(x)\n        \n        # Apply Squeeze-and-Excitation block for channel attention\n        x = self.se_block(x)\n        \n        # Pass through additional convolutional layers\n        x = self.conv_extra(x)\n        \n        # Apply adaptive pooling\n        x = self.adaptive_pool(x)  # Output shape will be (512, 1, 1)\n        \n        # Flatten the tensor for fully connected layers\n        x = torch.flatten(x, 1)  # Output shape will be (512,)\n        \n        # Pass through fully connected layers\n        x = self.fc_layers(x)\n        \n        return x\nmodel = ComplexResNet50(num_classes=10).to('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Instantiate the model\nif os.path.exists('./model.pt'):\n    model.load_state_dict(torch.load('./model.pt'))\n#     model = torch.load('./model.pt').to(device)\n# else:\n#     model = ComplexResNet50(num_classes=10).to(device)\n\n    # Load the best model\n# model = ComplexResNet50(num_classes=10).to(device)\n\n\n\n# Define custom WarmupCosineLR scheduler\nclass WarmupCosineLR(torch.optim.lr_scheduler._LRScheduler):\n    def __init__(self, optimizer, warmup_epochs, max_epochs, eta_min=0, last_epoch=-1):\n        self.warmup_epochs = warmup_epochs\n        self.max_epochs = max_epochs\n        self.eta_min = eta_min\n        super(WarmupCosineLR, self).__init__(optimizer, last_epoch)\n    \n    def get_lr(self):\n        if self.last_epoch < self.warmup_epochs:\n            # Linear warm-up\n            return [base_lr * (self.last_epoch + 1) / self.warmup_epochs for base_lr in self.base_lrs]\n        else:\n            # Cosine annealing\n            return [self.eta_min + (base_lr - self.eta_min) * 0.5 * \n                    (1 + math.cos(math.pi * (self.last_epoch - self.warmup_epochs) / (self.max_epochs - self.warmup_epochs)))\n                    for base_lr in self.base_lrs]\n\n# Use cuda if you have gpu\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n# Define model, optimizer, loss function\nmodel = ComplexResNet50(num_classes=10).to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\ncriterion = nn.CrossEntropyLoss().to(device)\nwarmup_epochs = 5\nscheduler = WarmupCosineLR(optimizer, warmup_epochs=warmup_epochs, max_epochs=num_epochs)\n\n# Train\nbest_epoch = 0\nbest_acc = 0.0\nfor epoch in range(1, num_epochs + 1):\n    print(\"-\" * 40)\n    print(f\"Epoch: {epoch}\")\n    epoch_loss = 0.0\n    epoch_corrects = 0\n    model.train()\n    for batch_in, batch_out in tqdm(train_loader):\n        batch_in = batch_in.to(device)\n        batch_out = batch_out.to(device)\n\n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        loss = criterion(y_pred, batch_out)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        epoch_loss += loss.item() * batch_in.size(0)\n        epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(train_loader.dataset)\n    epoch_acc = epoch_corrects.double() / len(train_loader.dataset)\n\n    print(f\"Train loss: {epoch_loss:.4f}\")\n    print(f\"Train acc: {epoch_acc:.4f}\")\n\n    # Validation\n    with torch.no_grad():\n        epoch_loss = 0.0\n        epoch_corrects = 0\n        model.eval()\n        for batch_in, batch_out in tqdm(valid_loader):\n            batch_in = batch_in.to(device)\n            batch_out = batch_out.to(device)\n\n            y_pred = model(batch_in)\n            preds = torch.argmax(y_pred, 1)\n            loss = criterion(y_pred, batch_out)\n\n            epoch_loss += loss.item() * batch_in.size(0)\n            epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(valid_loader.dataset)\n    epoch_acc = epoch_corrects.double() / len(valid_loader.dataset)\n    \n    scheduler.step()\n\n    print(f\"Valid loss: {epoch_loss:.4f}\")\n    print(f\"Valid acc: {epoch_acc:.4f}\")\n\n    if epoch_acc >= best_acc:\n        best_epoch = epoch\n        best_acc = epoch_acc\n        torch.save(model.state_dict(), \"./model.pt\")  # Save model state dict\n\n    print(f\"Best epoch: {best_epoch}\")\n    print(f\"Best acc: {best_acc:.4f}\")\n    print(\"-\" * 40)\n\n# Load the best model\nmodel.load_state_dict(torch.load('./model.pt'))\nmodel.to(device)\n\n# # Test Performance (Assuming test dataset is similar to validation dataset)\n# # Since CIFAR10 has a predefined test set, we can use it to evaluate the model\n# test_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n# with torch.no_grad():\n#     epoch_corrects = 0\n#     model.eval()\n#     for batch_in, batch_out in tqdm(test_loader):\n#         batch_in = batch_in.to(device)\n#         batch_out = batch_out.to(device)\n\n#         y_pred = model(batch_in)\n#         preds = torch.argmax(y_pred, 1)\n\n#         epoch_corrects += torch.sum(preds == batch_out.data)\n\n# test_acc = epoch_corrects.double() / len(test_loader.dataset)\n# print(f\"Test acc: {test_acc:.4f}\")\n\n# assert test_acc >= 0.95, \"Test performance is less than 0.95\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-18T12:59:23.857960Z","iopub.execute_input":"2024-10-18T12:59:23.858560Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')\n    \n# Make a list of data filenames for dataset\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))\n\n# Define dataset\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img\n\n# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')\n# Load the best model\nmodel = ComplexResNet50(num_classes=10).to(device)\nmodel.load_state_dict(torch.load('./model.pt'))\n# model = torch.load('model.pt')\n# model = model.to(device)\n# Get the result\nresult = []\n\nwith torch.no_grad():\n    model.eval()\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        \n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\nsample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')\n\nsample['label'] = result\nsample.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'submission.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}