{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.datasets as datasets\nfrom torchvision.models import resnet101\nfrom torchvision import transforms\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport torchinfo\nimport random\nimport os\n\n# Set seed\nrandom_seed = 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)\n\n# Hyperparameters\nbatch_size = 128  # Increased for better stability\nlearning_rate = 1e-3  # Higher starting learning rate\nnum_epochs = 100\nimg_size = 128\nwarmup_epochs = 5\nweight_decay = 1e-3  # Increased weight decay\n\n# Custom CutOut augmentation\nclass CutOut(object):\n    def __init__(self, length):\n        self.length = length\n\n    def __call__(self, img):\n        h, w = img.shape[1], img.shape[2]\n        mask = np.ones((h, w), np.float32)\n        y = np.random.randint(h)\n        x = np.random.randint(w)\n        y1 = np.clip(y - self.length // 2, 0, h)\n        y2 = np.clip(y + self.length // 2, 0, h)\n        x1 = np.clip(x - self.length // 2, 0, w)\n        x2 = np.clip(x + self.length // 2, 0, w)\n        mask[y1: y2, x1: x2] = 0.\n        img = img * torch.tensor(mask, dtype=torch.float32).unsqueeze(0)\n        return img\n\n# CutMix function\ndef cutmix(data, targets, alpha=1.0):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    lam = np.clip(np.random.beta(alpha, alpha), 0.3, 0.7)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    data[:, :, bbx1:bbx2, bby1:bby2] = shuffled_data[:, :, bbx1:bbx2, bby1:bby2]\n    return data, targets, shuffled_targets, lam\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    \n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n    \n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    \n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    \n    return bbx1, bby1, bbx2, bby2\n\n# Define transforms\ntrain_transforms = transforms.Compose([\n    transforms.Resize(img_size),\n    transforms.RandomRotation(20),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.ToTensor(),\n    CutOut(length=16),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ntest_transforms = transforms.Compose([\n    transforms.Resize(img_size),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Load CIFAR10 dataset\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n\n# Squeeze-and-Excitation (SE) Block\nclass SEBlock(nn.Module):\n    def __init__(self, channel, reduction=16):\n        super(SEBlock, self).__init__()\n        self.fc1 = nn.Linear(channel, channel // reduction, bias=False)\n        self.fc2 = nn.Linear(channel // reduction, channel, bias=False)\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = torch.mean(x, dim=(2, 3), keepdim=True)\n        y = y.view(b, c)\n        y = torch.relu(self.fc1(y))\n        y = torch.sigmoid(self.fc2(y)).view(b, c, 1, 1)\n        return x * y\n\n# Define the model\nclass ComplexResNet101(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ComplexResNet101, self).__init__()\n        self.model = resnet101(weights=None)\n        self.in_features = self.model.fc.in_features\n        self.model.fc = nn.Sequential(\n            nn.Dropout(0.5),\n            nn.Linear(self.in_features, 512),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, num_classes)\n        )\n        self.se_block = SEBlock(self.in_features)\n\n    def forward(self, x):\n        x = self.model.conv1(x)\n        x = self.model.bn1(x)\n        x = self.model.relu(x)\n        x = self.model.maxpool(x)\n\n        x = self.model.layer1(x)\n        x = self.model.layer2(x)\n        x = self.model.layer3(x)\n        x = self.se_block(self.model.layer4(x))\n        \n        x = self.model.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.model.fc(x)\n        return x\n\n# Instantiate the model and optimizer\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nmodel = ComplexResNet101(num_classes=10).to(device)\noptimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n\n# Learning rate scheduler with warmup and cosine annealing\nscheduler_steplr = torch.optim.lr_scheduler.LambdaLR(\n    optimizer, \n    lr_lambda=lambda epoch: epoch / warmup_epochs if epoch < warmup_epochs else 1.\n)\nscheduler_cosine = CosineAnnealingLR(optimizer, T_max=num_epochs - warmup_epochs)\n\n# Loss function with label smoothing\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1).to(device)\n\n# Training loop with CutMix and gradient clipping\nbest_epoch = 0\nbest_acc = 0.0\nfor epoch in range(1, num_epochs + 1):\n    print(f\"Epoch {epoch}/{num_epochs}\")\n    \n    model.train()\n    epoch_loss = 0.0\n    epoch_corrects = 0\n    for batch_in, batch_out in tqdm(train_loader):\n        batch_in, batch_out = batch_in.to(device), batch_out.to(device)\n        \n        # Use both CutOut and CutMix simultaneously for stronger augmentation\n        if np.random.rand(1) < 0.5:\n            batch_in, targets, shuffled_targets, lam = cutmix(batch_in, batch_out)\n            preds = model(batch_in)\n            loss = lam * criterion(preds, targets) + (1 - lam) * criterion(preds, shuffled_targets)\n        else:\n            preds = model(batch_in)\n            loss = criterion(preds, batch_out)\n        \n        optimizer.zero_grad()\n        loss.backward()\n\n        # Gradient clipping to prevent exploding gradients\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n\n        optimizer.step()\n\n        epoch_loss += loss.item() * batch_in.size(0)\n        epoch_corrects += torch.sum(torch.argmax(preds, 1) == batch_out)\n\n    epoch_loss /= len(train_loader.dataset)\n    epoch_acc = epoch_corrects.float() / len(train_loader.dataset)\n    print(f\"Train loss: {epoch_loss:.4f}, Train acc: {epoch_acc:.4f}\")\n    \n    # Validation\n    model.eval()\n    with torch.no_grad():\n        epoch_loss = 0.0\n        epoch_corrects = 0\n        for batch_in, batch_out in tqdm(valid_loader):\n            batch_in, batch_out = batch_in.to(device), batch_out.to(device)\n            preds = model(batch_in)\n            loss = criterion(preds, batch_out)\n            epoch_loss += loss.item() * batch_in.size(0)\n            epoch_corrects += torch.sum(torch.argmax(preds, 1) == batch_out)\n\n        epoch_loss /= len(valid_loader.dataset)\n        epoch_acc = epoch_corrects.float() / len(valid_loader.dataset)\n        \n    print(f\"Valid loss: {epoch_loss:.4f}, Valid acc: {epoch_acc:.4f}\")\n\n    # Update best model\n    if epoch_acc > best_acc:\n        best_acc = epoch_acc\n        best_epoch = epoch\n        torch.save(model.state_dict(), 'best_model.pth')\n\n    # Step the schedulers\n    scheduler_steplr.step() if epoch <= warmup_epochs else scheduler_cosine.step()\n\n    print(f\"Best epoch: {best_epoch}, Best acc: {best_acc:.4f}\")\n\n# Print the best accuracy at the end of training\nprint(f\"Training complete! Best accuracy: {best_acc:.4f} at epoch {best_epoch}\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-06T14:40:01.350785Z","iopub.execute_input":"2024-10-06T14:40:01.351736Z","iopub.status.idle":"2024-10-06T20:31:33.932891Z","shell.execute_reply.started":"2024-10-06T14:40:01.351683Z","shell.execute_reply":"2024-10-06T20:31:33.931927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')\n    \n# Make a list of data filenames for dataset\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))\n\n# Define dataset\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img\n\n# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')\n# Instantiate the model architecture\nmodel = ComplexResNet101(num_classes=10).to(device)\n\n# Load the state dictionary (weights) into the model\nmodel.load_state_dict(torch.load('/kaggle/working/best_model.pth'))\n\n# Set the model to evaluation mode\nmodel.eval()\n\n# Perform inference as before\nresult = []\n\nwith torch.no_grad():\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\n\n# Save results to submission file\nsample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')\nsample['label'] = result\nsample.to_csv('./submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T20:47:17.003980Z","iopub.execute_input":"2024-10-06T20:47:17.004479Z","iopub.status.idle":"2024-10-06T20:58:52.981318Z","shell.execute_reply.started":"2024-10-06T20:47:17.004432Z","shell.execute_reply":"2024-10-06T20:58:52.980451Z"},"trusted":true},"execution_count":null,"outputs":[]}]}