{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nfrom torchvision import models\nfrom tqdm import tqdm\n\nbatch_size = 128\nlearning_rate = 1e-3\nnum_epochs = 100  # Train for more epochs\nimg_size = 128\n\n# Check if multiple GPUs are available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nif torch.cuda.device_count() > 1:\n    print(f\"Using {torch.cuda.device_count()} GPUs!\")\nelse:\n    print(f\"Using {torch.cuda.device_count()} GPU.\")\n\n# Define aggressive data augmentations\ntrain_transforms = transforms.Compose([\n    transforms.Resize((img_size, img_size)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomRotation(30),\n    transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.2),  # Strong color augmentation\n    transforms.RandomResizedCrop(size=img_size, scale=(0.8, 1.0)),\n    transforms.RandomGrayscale(p=0.1),  # Occasionally convert image to grayscale\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ntest_transforms = transforms.Compose([\n    transforms.Resize((img_size, img_size)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Load CIFAR-10 dataset\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n\n# Define a modified ResNet50 without pre-trained weights\nclass ModifiedResNet50(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ModifiedResNet50, self).__init__()\n        self.resnet = models.resnet50(weights=None)  # Do not use pre-trained weights\n        self.resnet.fc = nn.Sequential(\n            nn.Dropout(0.5),  # Add dropout to prevent overfitting\n            nn.Linear(self.resnet.fc.in_features, num_classes)\n        )\n\n    def forward(self, x):\n        return self.resnet(x)\n\nmodel = ModifiedResNet50(num_classes=10)\n\n# Use DataParallel to distribute model across 2 GPUs\nif torch.cuda.device_count() > 1:\n    model = nn.DataParallel(model, device_ids=[0, 1])\n\n# Move model to GPU(s)\nmodel = model.to(device)\n\n# Define optimizer with weight decay (L2 regularization) and learning rate scheduler\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-4)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, factor=0.5, verbose=True)  # Reduce LR on plateau\ncriterion = nn.CrossEntropyLoss()\n\n# Train the model\nbest_valid_acc = 0.0\n\nfor epoch in range(1, num_epochs + 1):\n    print(f\"Epoch {epoch}\")\n\n    # Training Phase\n    model.train()\n    train_loss, train_corrects, train_total = 0.0, 0, 0\n\n    with tqdm(train_loader, unit=\"batch\") as tepoch:\n        for batch_in, batch_out in tepoch:\n            tepoch.set_description(f\"Train Epoch {epoch}\")\n\n            optimizer.zero_grad()\n\n            batch_in, batch_out = batch_in.to(device), batch_out.to(device)\n            y_pred = model(batch_in)\n            loss = criterion(y_pred, batch_out)\n            loss.backward()\n\n            optimizer.step()\n\n            preds = torch.argmax(y_pred, 1)\n            train_loss += loss.item() * batch_in.size(0)\n            train_corrects += torch.sum(preds == batch_out).item()\n            train_total += batch_in.size(0)\n\n            current_train_loss = train_loss / train_total\n            current_train_acc = train_corrects / train_total\n            tepoch.set_postfix(loss=current_train_loss, accuracy=current_train_acc)\n\n    train_loss /= train_total\n    train_acc = train_corrects / train_total\n    print(f'Epoch {epoch}, Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}')\n\n    # Validation Phase\n    model.eval()\n    valid_loss, valid_corrects, valid_total = 0.0, 0, 0\n\n    with tqdm(valid_loader, unit=\"batch\") as tepoch:\n        for batch_in, batch_out in tepoch:\n            tepoch.set_description(f\"Validation Epoch {epoch}\")\n\n            batch_in, batch_out = batch_in.to(device), batch_out.to(device)\n            with torch.no_grad():\n                y_pred = model(batch_in)\n                loss = criterion(y_pred, batch_out)\n\n            preds = torch.argmax(y_pred, 1)\n            valid_loss += loss.item() * batch_in.size(0)\n            valid_corrects += torch.sum(preds == batch_out).item()\n            valid_total += batch_in.size(0)\n\n            current_valid_loss = valid_loss / valid_total\n            current_valid_acc = valid_corrects / valid_total\n            tepoch.set_postfix(loss=current_valid_loss, accuracy=current_valid_acc)\n\n    valid_loss /= valid_total\n    valid_acc = valid_corrects / valid_total\n    print(f'Epoch {epoch}, Valid Loss: {valid_loss:.4f}, Valid Acc: {valid_acc:.4f}')\n\n    # Check for improvement in validation accuracy\n    if valid_acc > best_valid_acc:\n        best_valid_acc = valid_acc\n        torch.save(model.state_dict(), f'best_model_epoch_{epoch}.pt')\n        print(f\"Model saved at epoch {epoch} with validation accuracy {valid_acc:.4f}\")\n\n    # Step the learning rate scheduler\n    scheduler.step(valid_acc)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-01T12:49:18.128859Z","iopub.execute_input":"2024-10-01T12:49:18.129232Z","iopub.status.idle":"2024-10-01T16:07:39.783468Z","shell.execute_reply.started":"2024-10-01T12:49:18.129195Z","shell.execute_reply":"2024-10-01T16:07:39.782262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nlearning_rate = 1e-4\nnum_epochs = 200\nimg_size = 128","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:07:47.87252Z","iopub.execute_input":"2024-10-01T16:07:47.873715Z","iopub.status.idle":"2024-10-01T16:07:47.878546Z","shell.execute_reply.started":"2024-10-01T16:07:47.873667Z","shell.execute_reply":"2024-10-01T16:07:47.877459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read csv\nimport pandas as pd\nsample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:08:09.009686Z","iopub.execute_input":"2024-10-01T16:08:09.010533Z","iopub.status.idle":"2024-10-01T16:08:09.506387Z","shell.execute_reply.started":"2024-10-01T16:08:09.010492Z","shell.execute_reply":"2024-10-01T16:08:09.505367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the form of data\nsample.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:08:13.164988Z","iopub.execute_input":"2024-10-01T16:08:13.165366Z","iopub.status.idle":"2024-10-01T16:08:13.184209Z","shell.execute_reply.started":"2024-10-01T16:08:13.165331Z","shell.execute_reply":"2024-10-01T16:08:13.183322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:14:28.794432Z","iopub.execute_input":"2024-10-01T16:14:28.794809Z","iopub.status.idle":"2024-10-01T16:17:12.3897Z","shell.execute_reply.started":"2024-10-01T16:14:28.794774Z","shell.execute_reply":"2024-10-01T16:17:12.388558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a list of data filenames for dataset\nimport os\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:19:24.604969Z","iopub.execute_input":"2024-10-01T16:19:24.605788Z","iopub.status.idle":"2024-10-01T16:19:25.633212Z","shell.execute_reply.started":"2024-10-01T16:19:24.605745Z","shell.execute_reply":"2024-10-01T16:19:25.63195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:19:29.394342Z","iopub.execute_input":"2024-10-01T16:19:29.395212Z","iopub.status.idle":"2024-10-01T16:19:29.401169Z","shell.execute_reply.started":"2024-10-01T16:19:29.395167Z","shell.execute_reply":"2024-10-01T16:19:29.400016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:19:32.065757Z","iopub.execute_input":"2024-10-01T16:19:32.066385Z","iopub.status.idle":"2024-10-01T16:19:32.071261Z","shell.execute_reply.started":"2024-10-01T16:19:32.066343Z","shell.execute_reply":"2024-10-01T16:19:32.070335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:19:34.973127Z","iopub.execute_input":"2024-10-01T16:19:34.973506Z","iopub.status.idle":"2024-10-01T16:19:34.978592Z","shell.execute_reply.started":"2024-10-01T16:19:34.973468Z","shell.execute_reply":"2024-10-01T16:19:34.977475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the best model\nimport torch\nfrom PIL import Image\n# Get the result\nresult = []\n\nwith torch.no_grad():\n    model.eval()\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        \n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\n\nsample['label'] = result\nsample.to_csv('./submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T16:29:19.744674Z","iopub.execute_input":"2024-10-01T16:29:19.74508Z","iopub.status.idle":"2024-10-01T16:38:01.247422Z","shell.execute_reply.started":"2024-10-01T16:29:19.745043Z","shell.execute_reply":"2024-10-01T16:38:01.246637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}