{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import DataLoader\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nimport torch.optim as optim\nimport numpy as np\nfrom PIL import Image\nimport random\nimport pandas as pd\n!pip install py7zr\nimport py7zr\nfrom io import BytesIO\nimport torch.optim.lr_scheduler as lr_scheduler\n\n# Define a basic residual block with dropout\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, stride=1):\n        super(ResidualBlock, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        self.dropout = nn.Dropout(p=0.3)  # Dropout to regularize\n\n        self.skip_connection = nn.Sequential()\n        if stride != 1 or in_channels != out_channels:\n            self.skip_connection = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels)\n            )\n\n    def forward(self, x):\n        identity = self.skip_connection(x)\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.dropout(out)  # Apply dropout\n        out += identity\n        out = self.relu(out)\n        return out\n\n# Define ResNet-34 for CIFAR-10\nclass ResNet34(nn.Module):\n    def __init__(self, block, num_blocks, num_classes=10):\n        super(ResNet34, self).__init__()\n        self.in_channels = 64\n\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512, num_classes)\n\n    def _make_layer(self, block, out_channels, blocks, stride):\n        strides = [stride] + [1] * (blocks - 1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.in_channels, out_channels, stride))\n            self.in_channels = out_channels\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.layer4(out)\n        out = self.avg_pool(out)\n        out = torch.flatten(out, 1)\n        out = self.fc(out)\n        return out\n\n# Function to instantiate the ResNet-34 model\ndef get_resnet34():\n    return ResNet34(ResidualBlock, [3, 4, 6, 3])\n\n# Custom CutOut transformation\nclass CutOut(object):\n    def __init__(self, n_holes, length):\n        self.n_holes = n_holes\n        self.length = length\n\n    def __call__(self, img):\n        h, w = img.size(1), img.size(2)\n        mask = np.ones((h, w), np.float32)\n\n        for n in range(self.n_holes):\n            y = random.randint(0, h)\n            x = random.randint(0, w)\n\n            y1 = np.clip(y - self.length // 2, 0, h)\n            y2 = np.clip(y + self.length // 2, 0, h)\n            x1 = np.clip(x - self.length // 2, 0, w)\n            x2 = np.clip(x + self.length // 2, 0, w)\n\n            mask[y1:y2, x1:x2] = 0.\n\n        mask = torch.from_numpy(mask).expand_as(img)\n        img = img * mask\n        return img\n\n# Transformation function using PyTorch's transforms\ntransform_train = transforms.Compose([\n    transforms.RandomHorizontalFlip(p=0.5),  # Horizontal flip\n    transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),  # Rotate, scale, and shift\n    transforms.RandomCrop(32, padding=4),  # Random crop with padding\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010)),\n    CutOut(n_holes=1, length=8),  # Use CutOut after ToTensor for regularization\n])\n\ntransform_test = transforms.Compose([\n    transforms.Resize((32, 32)),  # Resize to 32x32\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010)),\n])\n\n# Example of applying the transformations to the CIFAR-10 dataset\ntrain_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)\ntest_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=256, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False)\n\n# Define device (GPU/CPU)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Create model, define loss function and optimizer\nmodel = get_resnet34().to(device)\ncriterion = nn.CrossEntropyLoss()\n\n# Use AdamW optimizer with weight decay\noptimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=1e-4)\n# Calculate steps per epoch\nsteps_per_epoch = len(train_loader)  # Number of batches in the training set\n\n# Cosine annealing scheduler with warmup\n\n# Define the OneCycleLR scheduler\nscheduler = lr_scheduler.OneCycleLR(\n    optimizer, \n    max_lr=0.001,        # The maximum learning rate after warmup\n    steps_per_epoch=steps_per_epoch,  # Total steps in one epoch (train dataset size / batch size)\n    epochs=80,           # Total number of epochs\n    pct_start=0.3,       # Warmup period (30% of the total steps)\n    anneal_strategy='cos',  # Cosine annealing after warmup\n    div_factor=25.0      # Initial learning rate will be max_lr / div_factor\n)\n\n# Training loop\ndef train(epoch):\n    model.train()\n    running_loss = 0.0\n    for batch_idx, (inputs, targets) in enumerate(train_loader):\n        inputs, targets = inputs.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n\n        # Update the learning rate at each batch step\n        scheduler.step()\n\n        running_loss += loss.item()\n        if batch_idx % 100 == 99:\n            print(f\"Epoch {epoch+1}, Batch {batch_idx+1}: Loss = {running_loss / 100:.4f}\")\n            running_loss = 0.0\n\ndef test(epoch):\n    model.eval()\n    test_loss = 0.0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs, targets) in enumerate(test_loader):\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            test_loss += loss.item()\n\n            _, predicted = outputs.max(1)\n            total += targets.size(0)\n            correct += predicted.eq(targets).sum().item()\n\n        print(f\"Epoch {epoch+1}: Test Loss = {test_loss / len(test_loader):.4f}, Accuracy = {100. * correct / total:.2f}%\")\n\n# Main training loop\nfor epoch in range(80):\n    train(epoch)\n    test(epoch)\n\n# Run the test for submission\n# Prediction process omitted for brevity; you can reuse your existing test set submission process.\n\n# Create submission file with predictions for test images from .7z archive\ntest_filenames = []\ntest_images = []\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    for name, file in z.readall().items():\n        if name.endswith('.png'):\n            img = Image.open(BytesIO(file.read()))\n            test_images.append(transform_test(img))\n            test_filenames.append(name)\n\ntest_images = torch.stack(test_images)\ntest_loader = DataLoader(test_images, batch_size=100, shuffle=False)\n\n# Prediction and CSV creation\nclasses = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\nresult = []\nwith torch.no_grad():\n    model.eval()\n    for inputs in tqdm(test_loader):\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        _, predicted = outputs.max(1)\n        result.extend(predicted.cpu().numpy())\n\n# Create submission DataFrame\nsubmission_df = pd.DataFrame({\n    'id': [os.path.basename(f).replace('.png', '') for f in test_filenames],  # Remove .png from filenames\n    'label': [classes[label] for label in result]\n})\n\n# Save submission file\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)\n\nprint(\"Submission file created successfully!\")\n","metadata":{},"execution_count":null,"outputs":[]}]}