{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.datasets as datasets\nfrom torchvision.models import resnet50\nfrom torchvision import transforms\n\nimport torchinfo\n\n# Hyperparameters\nbatch_size = 64\nlearning_rate = 1e-4\nnum_epochs = 120\nimg_size = 128\n\n# Set seed for reproducibility\nimport os\nimport random\nimport numpy as np\nimport torch\n\nrandom_seed = 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)\n\n# Define transforms with additional augmentations\ntrain_transforms = transforms.Compose(\n    [\n        transforms.RandomResizedCrop(img_size, scale=(0.8, 1.0)),  # Random crop and resize\n        transforms.RandomRotation(20),                             # Random rotation up to 20 degrees\n        transforms.RandomHorizontalFlip(0.5),                      # Random horizontal flip\n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),  # Random color jitter\n        transforms.RandomGrayscale(p=0.1),                         # Convert to grayscale with 10% probability\n        transforms.RandomAffine(15, translate=(0.1, 0.1)),         # Random affine transformation\n        transforms.ToTensor(),\n        transforms.RandomErasing(p=0.3, scale=(0.02, 0.2)),        # Random erasing for occlusion\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),  # Normalize\n    ]\n)\n\n# Test transforms remain the same\ntest_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),  # Normalize\n    ]\n)\n\n# Load data\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)  # Shuffle enabled for training\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n\n\n# Squeeze-and-Excitation Block for attention mechanism\nclass SEBlock(nn.Module):\n    def __init__(self, in_channels, reduction=16):\n        super(SEBlock, self).__init__()\n        self.fc1 = nn.Conv2d(in_channels, in_channels // reduction, kernel_size=1)\n        self.fc2 = nn.Conv2d(in_channels // reduction, in_channels, kernel_size=1)\n\n    def forward(self, x):\n        out = torch.mean(x, dim=[2, 3], keepdim=True)\n        out = torch.relu(self.fc1(out))\n        out = torch.sigmoid(self.fc2(out))\n        return x * out\n\n\n# Define the custom complex model based on ResNet50\nclass ComplexResNet50(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ComplexResNet50, self).__init__()\n        \n        # Initial convolutional block before ResNet50 for richer feature extraction\n        self.pre_conv = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=3, padding=1),  # Output 64 channels, as expected by ResNet50\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Load ResNet50 without pre-trained weights\n        self.resnet50 = resnet50(weights=None)\n        \n        # Skip the initial conv, bn1, relu, and maxpool layers from ResNet50\n        self.resnet50 = nn.Sequential(\n            self.resnet50.layer1,  # Start from layer1 (expects 64 channels)\n            self.resnet50.layer2,\n            self.resnet50.layer3,\n            self.resnet50.layer4\n        )\n        \n        # Add an SE block for attention\n        self.se_block = SEBlock(2048)\n        \n        # Additional convolutional layers after ResNet50\n        self.conv_extra = nn.Sequential(\n            nn.Conv2d(2048, 1024, kernel_size=3, padding=1),\n            nn.BatchNorm2d(1024),\n            nn.ReLU(),\n            nn.Conv2d(1024, 512, kernel_size=3, padding=1),\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Use AdaptiveAvgPool2d to dynamically handle varying input sizes\n        self.adaptive_pool = nn.AdaptiveAvgPool2d((1, 1))  # This will output (512, 1, 1)\n        \n        # Fully connected layers for classification\n        self.fc_layers = nn.Sequential(\n            nn.Linear(512, 1024),  # Adjusted size based on the adaptive pooling output\n            nn.ReLU(),\n            nn.Dropout(0.5),  # Dropout for regularization\n            nn.Linear(1024, 512),\n            nn.ReLU(),\n            nn.BatchNorm1d(512),  # Batch normalization for stability\n            nn.Linear(512, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.pre_conv(x)\n        x = self.resnet50(x)\n        x = self.se_block(x)\n        x = self.conv_extra(x)\n        x = self.adaptive_pool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc_layers(x)\n        return x\n\n\n# Instantiate the model\nmodel = ComplexResNet50(num_classes=10)\n\n# Use cuda if available\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nmodel = model.to(device)\n\n# Define optimizer, loss function, and scheduler\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\ncriterion = nn.CrossEntropyLoss().to(device)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n\n# Training loop\nbest_epoch = 0\nbest_acc = 0.0\nfor epoch in range(1, num_epochs + 1):\n    print(\"-\" * 40)\n    print(f\"Epoch: {epoch}\")\n    \n    # Training phase\n    epoch_loss = 0.0\n    epoch_corrects = 0\n    model.train()\n    for batch_in, batch_out in tqdm(train_loader):\n        batch_in = batch_in.to(device)\n        batch_out = batch_out.to(device)\n\n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        loss = criterion(y_pred, batch_out)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        epoch_loss += loss.item() * batch_in.size(0)\n        epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(train_loader.dataset)\n    epoch_acc = epoch_corrects / len(train_loader.dataset)\n\n    print(f\"Train loss: {epoch_loss:.4f}\")\n    print(f\"Train acc: {epoch_acc:.4f}\")\n\n    # Validation phase\n    with torch.no_grad():\n        epoch_loss = 0.0\n        epoch_corrects = 0\n        model.eval()\n        for batch_in, batch_out in tqdm(valid_loader):\n            batch_in = batch_in.to(device)\n            batch_out = batch_out.to(device)\n\n            y_pred = model(batch_in)\n            preds = torch.argmax(y_pred, 1)\n            loss = criterion(y_pred, batch_out)\n\n            epoch_loss += loss.item() * batch_in.size(0)\n            epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(valid_loader.dataset)\n    epoch_acc = epoch_corrects / len(valid_loader.dataset)\n    \n    scheduler.step()\n\n    print(f\"Valid loss: {epoch_loss:.4f}\")\n    print(f\"Valid acc: {epoch_acc:.4f}\")\n\n    # Save the best model\n    if epoch_acc >= best_acc:\n        best_epoch = epoch\n        best_acc = epoch_acc\n        torch.save(model, \"./model.pt\")\n\n    print(f\"Best epoch: {best_epoch}\")\n    print(f\"Best acc: {best_acc:.4f}\")\n    print(\"-\" * 40)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-30T15:05:11.924170Z","iopub.execute_input":"2024-09-30T15:05:11.924779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')\n    \n# Make a list of data filenames for dataset\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))\n\n# Define dataset\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img\n\n# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')\n# Load the best model\nmodel = torch.load('./model.pt').to(device)\n\n# Get the result\nresult = []\n\nwith torch.no_grad():\n    model.eval()\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        \n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\n\nsample['label'] = result\nsample.to_csv('./predict.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}