{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nimport torch.optim as optim\nfrom PIL import Image\nimport numpy as np\nimport random\n!pip install py7zr\nimport py7zr\nfrom io import BytesIO\n\n# Define a basic residual block\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, stride=1):\n        super(ResidualBlock, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n        self.skip_connection = nn.Sequential()\n        if stride != 1 or in_channels != out_channels:\n            self.skip_connection = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels)\n            )\n\n    def forward(self, x):\n        identity = self.skip_connection(x)\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out += identity  # Residual connection\n        out = self.relu(out)\n        return out\n\n# Define ResNet for CIFAR-10\nclass ResNet(nn.Module):\n    def __init__(self, block, num_blocks, num_classes=10):\n        super(ResNet, self).__init__()\n        self.in_channels = 64\n\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n\n        # Global average pooling\n        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512, num_classes)\n\n    def _make_layer(self, block, out_channels, blocks, stride):\n        strides = [stride] + [1] * (blocks - 1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.in_channels, out_channels, stride))\n            self.in_channels = out_channels\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.layer4(out)\n\n        # Apply global average pooling\n        out = self.avg_pool(out)\n\n        # Flatten the output\n        out = torch.flatten(out, 1)\n        out = self.fc(out)\n        return out\n\n# Create ResNet18 instance\ndef ResNet18():\n    return ResNet(ResidualBlock, [2, 2, 2, 2])\n\n# Custom CutOut transformation\nclass CutOut(object):\n    def __init__(self, n_holes, length):\n        self.n_holes = n_holes\n        self.length = length\n\n    def __call__(self, img):\n        # Check if img is a tensor and convert to numpy array if needed\n        if isinstance(img, torch.Tensor):\n            img = img.permute(1, 2, 0).numpy()  # Convert to HWC format\n\n        (h, w, _) = img.shape  # Unpack height, width, and channels\n        mask = np.ones((h, w), np.float32)\n\n        for n in range(self.n_holes):\n            y = random.randint(0, h)\n            x = random.randint(0, w)\n\n            y1 = np.clip(y - self.length // 2, 0, h)\n            y2 = np.clip(y + self.length // 2, 0, h)\n            x1 = np.clip(x - self.length // 2, 0, w)\n            x2 = np.clip(x + self.length // 2, 0, w)\n\n            mask[y1:y2, x1:x2] = 0.\n\n        # Apply mask to the image\n        img = img * mask[:, :, np.newaxis]  # Make sure to maintain channel dimensions\n        img = torch.tensor(img).permute(2, 0, 1)  # Convert back to CHW format\n        return img\n\n# Data preprocessing and augmentation with CutOut\n\n\ntransform_train = transforms.Compose([\n    transforms.RandomAffine(degrees=0, translate=(0.1, 0.1), scale=(0.8, 1.2)),\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    CutOut(n_holes=1, length=20),  # Use CutOut instead of Random Erasing\n])\n\n\ntransform_test = transforms.Compose([\n    transforms.Resize((32, 32)),  # Resize images to match model input\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n])\n\n# Load CIFAR-10 dataset\nfull_train_dataset = torchvision.datasets.CIFAR10(root='/kaggle/working', train=True, download=True, transform=transform_train)\n\n# Split the dataset into training and validation sets (80/20 split)\ntrain_size = int(0.8 * len(full_train_dataset))\nval_size = len(full_train_dataset) - train_size\ntrain_dataset, val_dataset = random_split(full_train_dataset, [train_size, val_size])\ntest_dataset = torchvision.datasets.CIFAR10(root='/kaggle/working', train=False, download=True, transform=transform_test)\n\ntrain_loader = DataLoader(train_dataset, batch_size=256, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=100, shuffle=False)\ntorchvision_test_loader = DataLoader(test_dataset, batch_size=100, shuffle=False)\n\n# Define device (GPU/CPU)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Create model, define loss function and optimizer\nmodel = ResNet18().to(device)\ncriterion = nn.CrossEntropyLoss()\n\n# Use AdamW optimizer with weight decay\noptimizer = optim.AdamW(model.parameters(), lr=0.0005, weight_decay=5e-4)\n\n# Cosine annealing scheduler with warmup\nscheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=20, T_mult=1)\n\n# Training function\ndef train(epoch):\n    model.train()\n    running_loss = 0.0\n    for batch_idx, (inputs, targets) in enumerate(train_loader):\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n        if batch_idx % 100 == 99:\n            print(f\"Epoch {epoch+1}, Batch {batch_idx+1}: Loss = {running_loss / 100:.4f}\")\n            running_loss = 0.0\n\n# Validation function\ndef val(epoch):\n    model.eval()\n    val_loss = 0.0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs, targets) in enumerate(val_loader):\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            val_loss += loss.item()\n\n            _, predicted = outputs.max(1)\n            total += targets.size(0)\n            correct += predicted.eq(targets).sum().item()\n\n        print(f\"Epoch {epoch+1}: Validation Loss = {val_loss / len(val_loader):.4f}, Accuracy = {100. * correct / total:.2f}%\")\n\n# Test function\ndef test():\n    model.eval()\n    test_loss = 0.0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs, targets) in enumerate(torchvision_test_loader):\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            test_loss += loss.item()\n\n            _, predicted = outputs.max(1)\n            total += targets.size(0)\n            correct += predicted.eq(targets).sum().item()\n\n        print(f\"Test Loss = {test_loss / len(torchvision_test_loader):.4f}, Accuracy = {100. * correct / total:.2f}%\")\n\n# Main training loop\nfor epoch in range(0, 80):\n    train(epoch)\n    val(epoch)\n    scheduler.step()\n\n# Run the test\ntest()\n\n# Create submission file with predictions for test images from .7z archive\ntest_filenames = []\ntest_images = []\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    for name, file in z.readall().items():\n        if name.endswith('.png'):\n            img = Image.open(BytesIO(file.read()))\n            test_images.append(transform_test(img))\n            test_filenames.append(name)\n\ntest_images = torch.stack(test_images)\ntest_loader = DataLoader(test_images, batch_size=100, shuffle=False)\n\n# Prediction and CSV creation\nclasses = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\nresult = []\nwith torch.no_grad():\n    model.eval()\n    for inputs in tqdm(test_loader):\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        _, predicted = outputs.max(1)\n        result.extend(predicted.cpu().numpy())\n\n# Create submission DataFrame\nsubmission_df = pd.DataFrame({\n    'id': [os.path.basename(f).replace('.png', '') for f in test_filenames],  # Remove .png from filenames\n    'label': [classes[label] for label in result]\n})\n\n# Save submission file\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)\n\nprint(\"Submission file created successfully!\")\n","metadata":{},"execution_count":null,"outputs":[]}]}