{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.datasets as datasets\nfrom torchvision.models import resnet50\nfrom torchvision import transforms","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.573105Z","iopub.execute_input":"2024-09-30T14:03:46.57381Z","iopub.status.idle":"2024-09-30T14:03:46.580086Z","shell.execute_reply.started":"2024-09-30T14:03:46.573769Z","shell.execute_reply":"2024-09-30T14:03:46.579003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nlearning_rate = 1e-4\nnum_epochs = 100\nimg_size = 128","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.582186Z","iopub.execute_input":"2024-09-30T14:03:46.582859Z","iopub.status.idle":"2024-09-30T14:03:46.591804Z","shell.execute_reply.started":"2024-09-30T14:03:46.582804Z","shell.execute_reply":"2024-09-30T14:03:46.590952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set seed\nimport os\nimport random\nimport numpy as np\nimport torch\n\nrandom_seed = 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.592908Z","iopub.execute_input":"2024-09-30T14:03:46.593247Z","iopub.status.idle":"2024-09-30T14:03:46.604343Z","shell.execute_reply.started":"2024-09-30T14:03:46.593183Z","shell.execute_reply":"2024-09-30T14:03:46.603424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read csv\nsample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.606069Z","iopub.execute_input":"2024-09-30T14:03:46.606376Z","iopub.status.idle":"2024-09-30T14:03:46.689278Z","shell.execute_reply.started":"2024-09-30T14:03:46.606345Z","shell.execute_reply":"2024-09-30T14:03:46.688293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the form of data\nsample.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.690559Z","iopub.execute_input":"2024-09-30T14:03:46.690975Z","iopub.status.idle":"2024-09-30T14:03:46.701293Z","shell.execute_reply.started":"2024-09-30T14:03:46.690927Z","shell.execute_reply":"2024-09-30T14:03:46.700308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define transforms\ntrain_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.RandomRotation(20),\n        transforms.RandomHorizontalFlip(0.5),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\ntest_transforms = transforms.Compose(\n    [\n        transforms.Resize(img_size),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.702699Z","iopub.execute_input":"2024-09-30T14:03:46.703107Z","iopub.status.idle":"2024-09-30T14:03:46.712625Z","shell.execute_reply.started":"2024-09-30T14:03:46.703069Z","shell.execute_reply":"2024-09-30T14:03:46.711696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load data\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:46.713901Z","iopub.execute_input":"2024-09-30T14:03:46.714226Z","iopub.status.idle":"2024-09-30T14:03:48.331207Z","shell.execute_reply.started":"2024-09-30T14:03:46.714194Z","shell.execute_reply":"2024-09-30T14:03:48.330403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get pretrained ResNet50 model and customize for CIFAR10 classification\n# class ResNet50(nn.Module):\n#     def __init__(self, num_classes):\n#         super(ResNet50, self).__init__()\n#         self.model = resnet50(weights=None)\n#         self.in_features = self.model.fc.in_features\n#         self.model.fc = nn.Linear(self.in_features, num_classes)\n        \n#     def forward(self, x):\n#         x = self.model(x)\n#         return x\nimport torch\nimport torch.nn as nn\nfrom torchvision.models import resnet50\n\n# Squeeze-and-Excitation Block for attention mechanism\nclass SEBlock(nn.Module):\n    def __init__(self, in_channels, reduction=16):\n        super(SEBlock, self).__init__()\n        self.fc1 = nn.Conv2d(in_channels, in_channels // reduction, kernel_size=1)\n        self.fc2 = nn.Conv2d(in_channels // reduction, in_channels, kernel_size=1)\n\n    def forward(self, x):\n        # Squeeze operation\n        out = torch.mean(x, dim=[2, 3], keepdim=True)\n        # Excitation operation\n        out = torch.relu(self.fc1(out))\n        out = torch.sigmoid(self.fc2(out))\n        return x * out\n\n# Define the custom complex model based on ResNet50\nclass ComplexResNet50(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ComplexResNet50, self).__init__()\n        \n        # Initial convolutional block before ResNet50 for richer feature extraction\n        self.pre_conv = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=3, padding=1),  # Output 64 channels, as expected by ResNet50\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Load ResNet50 without pre-trained weights\n        self.resnet50 = resnet50(weights=None)\n        \n        # Skip the initial conv, bn1, relu, and maxpool layers from ResNet50\n        self.resnet50 = nn.Sequential(\n            self.resnet50.layer1,  # Start from layer1 (expects 64 channels)\n            self.resnet50.layer2,\n            self.resnet50.layer3,\n            self.resnet50.layer4\n        )\n        \n        # Add an SE block for attention\n        self.se_block = SEBlock(2048)\n        \n        # Additional convolutional layers after ResNet50\n        self.conv_extra = nn.Sequential(\n            nn.Conv2d(2048, 1024, kernel_size=3, padding=1),\n            nn.BatchNorm2d(1024),\n            nn.ReLU(),\n            nn.Conv2d(1024, 512, kernel_size=3, padding=1),\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Use AdaptiveAvgPool2d to dynamically handle varying input sizes\n        self.adaptive_pool = nn.AdaptiveAvgPool2d((1, 1))  # This will output (512, 1, 1)\n        \n        # Fully connected layers for classification\n        self.fc_layers = nn.Sequential(\n            nn.Linear(512, 1024),  # Adjusted size based on the adaptive pooling output\n            nn.ReLU(),\n            nn.Dropout(0.5),  # Dropout for regularization\n            nn.Linear(1024, 512),\n            nn.ReLU(),\n            nn.BatchNorm1d(512),  # Batch normalization for stability\n            nn.Linear(512, num_classes)\n        )\n\n    def forward(self, x):\n        # Pass through the initial convolutional block\n        x = self.pre_conv(x)\n        \n        # Pass through ResNet50 backbone (starting from layer1)\n        x = self.resnet50(x)\n        \n        # Apply Squeeze-and-Excitation block for channel attention\n        x = self.se_block(x)\n        \n        # Pass through additional convolutional layers\n        x = self.conv_extra(x)\n        \n        # Apply adaptive pooling\n        x = self.adaptive_pool(x)  # Output shape will be (512, 1, 1)\n        \n        # Flatten the tensor for fully connected layers\n        x = torch.flatten(x, 1)  # Output shape will be (512,)\n        \n        # Pass through fully connected layers\n        x = self.fc_layers(x)\n        \n        return x\n\n# Instantiate the model\nmodel = ComplexResNet50(num_classes=10)\n\n# Test the model with an example input\n# batch_in = torch.randn(1, 3, 224, 224)  # Example input with batch size 1, 3 channels, 224x224 image\n# output = model(batch_in)\n# print(output.shape)  # Check the output shape\n","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:48.333674Z","iopub.execute_input":"2024-09-30T14:03:48.333993Z","iopub.status.idle":"2024-09-30T14:03:48.930217Z","shell.execute_reply.started":"2024-09-30T14:03:48.33396Z","shell.execute_reply":"2024-09-30T14:03:48.929424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use cuda if you have gpu\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n# Define model, optimizer, loss function\n# model = ResNet50(num_classes=10).to(device)\nmodel = ComplexResNet50(num_classes=10).to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\ncriterion = nn.CrossEntropyLoss().to(device)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:48.931638Z","iopub.execute_input":"2024-09-30T14:03:48.931968Z","iopub.status.idle":"2024-09-30T14:03:49.636047Z","shell.execute_reply.started":"2024-09-30T14:03:48.931932Z","shell.execute_reply":"2024-09-30T14:03:49.634706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchinfo\n# model = ComplexResNet50(num_classes=10)\n# torchinfo.summary(model, input_size=(1, 3, 224, 224), verbose=1, col_names=[\"input_size\", \"output_size\", \"kernel_size\", \"num_params\", \"params_percent\", \"trainable\"])","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:49.63785Z","iopub.execute_input":"2024-09-30T14:03:49.638267Z","iopub.status.idle":"2024-09-30T14:03:49.643508Z","shell.execute_reply.started":"2024-09-30T14:03:49.638206Z","shell.execute_reply":"2024-09-30T14:03:49.6422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\nbest_epoch = 0\nbest_acc = 0.0\nfor epoch in range(1, num_epochs + 1):\n    print(\"-\" * 40)\n    print(f\"Epoch: {epoch}\")\n    epoch_loss = 0.0\n    epoch_corrects = 0\n    model.train()\n    for batch_in, batch_out in tqdm(train_loader):\n        batch_in = batch_in.to(device)\n        batch_out = batch_out.to(device)\n\n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        loss = criterion(y_pred, batch_out)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        epoch_loss += loss.item() * batch_in.size(0)\n        epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(train_loader.dataset)\n    epoch_acc = epoch_corrects / len(train_loader.dataset)\n\n    print(f\"Train loss: {epoch_loss:.4f}\")\n    print(f\"Train acc: {epoch_acc:.4f}\")\n\n# Validation\n    with torch.no_grad():\n        epoch_loss = 0.0\n        epoch_corrects = 0\n        model.eval()\n        for batch_in, batch_out in tqdm(valid_loader):\n            batch_in = batch_in.to(device)\n            batch_out = batch_out.to(device)\n\n            y_pred = model(batch_in)\n            preds = torch.argmax(y_pred, 1)\n            loss = criterion(y_pred, batch_out)\n\n            epoch_loss += loss.item() * batch_in.size(0)\n            epoch_corrects += torch.sum(preds == batch_out.data)\n\n    epoch_loss = epoch_loss / len(valid_loader.dataset)\n    epoch_acc = epoch_corrects / len(valid_loader.dataset)\n    \n    scheduler.step()\n\n    print(f\"Valid loss: {epoch_loss:.4f}\")\n    print(f\"Valid acc: {epoch_acc:.4f}\")\n\n    if epoch_acc >= best_acc:\n        best_epoch = epoch\n        best_acc = epoch_acc\n        torch.save(model, \"./model.pt\")\n\n    print(f\"Best epoch: {best_epoch}\")\n    print(f\"Best acc: {best_acc:.4f}\")\n    print(\"-\" * 40)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T14:03:49.644861Z","iopub.execute_input":"2024-09-30T14:03:49.64516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a list of data filenames for dataset\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define dataset\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the best model\nmodel = torch.load('./model.pt').to(device)\n\n# Get the result\nresult = []\n\nwith torch.no_grad():\n    model.eval()\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        \n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\n\nsample['label'] = result\nsample.to_csv('./predict.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}