{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.datasets as datasets\nfrom torchvision.models import resnet50\nfrom torchvision import transforms\n\nimport torchinfo\n\n\nbatch_size = 128\nlearning_rate = 1e-4\nnum_epochs = 50\nimg_size = 128\n\n# Set seed\nimport os\nimport random\nimport numpy as np\nimport torch\n\nrandom_seed = 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)\n\n\n# Define transforms\ntrain_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.RandomRotation(20),\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\ntest_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\n\n\n# Load data\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n\n\n# # Get pretrained ResNet50 model and customize for CIFAR10 classification\n# class ResNet50(nn.Module):\n#     def __init__(self, num_classes):\n#         super(ResNet50, self).__init__()\n#         self.model = resnet50(weights=None)\n#         self.in_features = self.model.fc.in_features\n#         self.model.fc = nn.Linear(self.in_features, num_classes)\n        \n#     def forward(self, x):\n#         x = self.model(x)\n#         return x\nimport torch\nimport torch.nn as nn\nfrom torchvision.models import resnet50\n\n# Squeeze-and-Excitation Block for attention mechanism\nclass SEBlock(nn.Module):\n    def __init__(self, in_channels, reduction=16):\n        super(SEBlock, self).__init__()\n        self.fc1 = nn.Conv2d(in_channels, in_channels // reduction, kernel_size=1)\n        self.fc2 = nn.Conv2d(in_channels // reduction, in_channels, kernel_size=1)\n\n    def forward(self, x):\n        # Squeeze operation\n        out = torch.mean(x, dim=[2, 3], keepdim=True)\n        # Excitation operation\n        out = torch.relu(self.fc1(out))\n        out = torch.sigmoid(self.fc2(out))\n        return x * out\n\n# Define the custom complex model based on ResNet50\nclass ComplexResNet50(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ComplexResNet50, self).__init__()\n        \n        # Initial convolutional block before ResNet50 for richer feature extraction\n        self.pre_conv = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=3, padding=1),  # Output 64 channels, as expected by ResNet50\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Load ResNet50 without pre-trained weights\n        self.resnet50 = resnet50(weights=None)\n        \n        # Skip the initial conv, bn1, relu, and maxpool layers from ResNet50\n        self.resnet50 = nn.Sequential(\n            self.resnet50.layer1,  # Start from layer1 (expects 64 channels)\n            self.resnet50.layer2,\n            self.resnet50.layer3,\n            self.resnet50.layer4\n        )\n        \n        # Add an SE block for attention\n        self.se_block = SEBlock(2048)\n        \n        # Additional convolutional layers after ResNet50\n        self.conv_extra = nn.Sequential(\n            nn.Conv2d(2048, 1024, kernel_size=3, padding=1),\n            nn.BatchNorm2d(1024),\n            nn.ReLU(),\n            nn.Conv2d(1024, 512, kernel_size=3, padding=1),\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Use AdaptiveAvgPool2d to dynamically handle varying input sizes\n        self.adaptive_pool = nn.AdaptiveAvgPool2d((1, 1))  # This will output (512, 1, 1)\n        \n        # Fully connected layers for classification\n        self.fc_layers = nn.Sequential(\n            nn.Linear(512, 1024),  # Adjusted size based on the adaptive pooling output\n            nn.ReLU(),\n            nn.Dropout(0.5),  # Dropout for regularization\n            nn.Linear(1024, 512),\n            nn.ReLU(),\n            nn.BatchNorm1d(512),  # Batch normalization for stability\n            nn.Linear(512, num_classes)\n        )\n\n    def forward(self, x):\n        # Pass through the initial convolutional block\n        x = self.pre_conv(x)\n        \n        # Pass through ResNet50 backbone (starting from layer1)\n        x = self.resnet50(x)\n        \n        # Apply Squeeze-and-Excitation block for channel attention\n        x = self.se_block(x)\n        \n        # Pass through additional convolutional layers\n        x = self.conv_extra(x)\n        \n        # Apply adaptive pooling\n        x = self.adaptive_pool(x)  # Output shape will be (512, 1, 1)\n        \n        # Flatten the tensor for fully connected layers\n        x = torch.flatten(x, 1)  # Output shape will be (512,)\n        \n        # Pass through fully connected layers\n        x = self.fc_layers(x)\n        \n        return x\n\n# Instantiate the model\nmodel = ComplexResNet50(num_classes=10)\n\n# Test the model with an example input\n# batch_in = torch.randn(1, 3, 224, 224)  # Example input with batch size 1, 3 channels, 224x224 image\n# output = model(batch_in)\n# print(output.shape)  # Check the output shape\n# Use cuda if you have gpu\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n# Define model, optimizer, loss function\n# model = ResNet50(num_classes=10).to(device)\nif os.path.exists('./model.pt'):\n    model = torch.load('./model.pt').to(device)\nelse:\n    model = ComplexResNet50(num_classes=10).to(device)\n    \noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\ncriterion = nn.CrossEntropyLoss().to(device)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n\n\n# Train\nbest_epoch = 0\nbest_acc = 0.0\nfor epoch in range(1, num_epochs + 1):\n    print(\"-\" * 40)\n    print(f\"Epoch: {epoch}\")\n    epoch_loss = 0.0\n    epoch_corrects = 0\n    model.train()\n    for batch_in, batch_out in tqdm(train_loader):\n        batch_in = batch_in.to(device)\n        batch_out = batch_out.to(device)\n\n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        loss = criterion(y_pred, batch_out)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        epoch_loss += loss.item() * batch_in.size(0)\n        epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(train_loader.dataset)\n    epoch_acc = epoch_corrects / len(train_loader.dataset)\n\n    print(f\"Train loss: {epoch_loss:.4f}\")\n    print(f\"Train acc: {epoch_acc:.4f}\")\n\n# Validation\n    with torch.no_grad():\n        epoch_loss = 0.0\n        epoch_corrects = 0\n        model.eval()\n        for batch_in, batch_out in tqdm(valid_loader):\n            batch_in = batch_in.to(device)\n            batch_out = batch_out.to(device)\n\n            y_pred = model(batch_in)\n            preds = torch.argmax(y_pred, 1)\n            loss = criterion(y_pred, batch_out)\n\n            epoch_loss += loss.item() * batch_in.size(0)\n            epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(valid_loader.dataset)\n    epoch_acc = epoch_corrects / len(valid_loader.dataset)\n    \n    scheduler.step()\n\n    print(f\"Valid loss: {epoch_loss:.4f}\")\n    print(f\"Valid acc: {epoch_acc:.4f}\")\n\n    if epoch_acc >= best_acc:\n        best_epoch = epoch\n        best_acc = epoch_acc\n        torch.save(model, \"./model.pt\")\n\n    print(f\"Best epoch: {best_epoch}\")\n    print(f\"Best acc: {best_acc:.4f}\")\n    print(\"-\" * 40)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-02T11:18:46.317684Z","iopub.execute_input":"2024-10-02T11:18:46.318088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')\n    \n# Make a list of data filenames for dataset\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))\n\n# Define dataset\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img\n\n# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')\n# Load the best model\nmodel = torch.load('./model.pt').to(device)\n# Get the result\nresult = []\n\nwith torch.no_grad():\n    model.eval()\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        \n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\nsample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')\nsample['label'] = result\nsample.to_csv('./submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')\n# sample['label'] = result\n# sample.to_csv('./submission.csv', index=False)\n# sample.to_csv('./submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}