{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-10-18T15:00:51.796611Z","iopub.execute_input":"2024-10-18T15:00:51.797770Z","iopub.status.idle":"2024-10-18T15:00:53.154822Z","shell.execute_reply.started":"2024-10-18T15:00:51.797712Z","shell.execute_reply":"2024-10-18T15:00:53.153488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport pandas as pd\nfrom PIL import Image\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR\nfrom torch.utils.data import Dataset\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport random","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from py7zr import unpack_7zarchive\nimport shutil\n\nshutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#shutil.rmtree('/kaggle/working/train')\n#shutil.rmtree('/kaggle/working/test')\n#os.remove('/kaggle/working/cifar10_resnet18.pth')\n#os.remove('/kaggle/working/submission.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Configurations\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        # Define the paths\ntrain_input_path = '../input/cifar-10/train.7z' \ntest_input_path = '../input/cifar-10/test.7z'  # Path to the .7z archive\nlabels_csv = \"../input/cifar-10/trainLabels.csv\"\n\n        # Directory of extracted files\ntrain_output_path = '/kaggle/working/train/'  \ntest_output_path = '/kaggle/working/test/'  # Path to the directory where you want to save the unpacked images\n\n\n#shutil.rmtree(train_output_path)\n#shutil.rmtree(test_output_path)\n\n\nos.makedirs(train_output_path)\nos.makedirs(test_output_path)\n\nbatch_size = 256 # previous: 64, 128. Yoy may also try 256\nnum_epochs = 500\nnum_classes = 10\n#learning_rate = 0.1 # previous: 3e-4 and 0.5. You may also try 0.1. \nwarmup_epochs = 20","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"shutil.unpack_archive(train_input_path, train_output_path)\nprint(f\"Unpacked files for training images are saved to {train_output_path}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"shutil.unpack_archive(test_input_path, test_output_path)\nprint(f\"Unpacked files for training images are saved to {test_output_path}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dir = '/kaggle/working/train/train'  \ntest_dir = '/kaggle/working/test/test'  ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\ndef show_rgb_image(image, title=None, conversion=cv2.COLOR_BGR2RGB):\n\n    # Converts from one colour space to the other. this is needed as RGB\n    # is not the default colour space for OpenCV\n    image = cv2.cvtColor(image, conversion)\n\n    # Show the image\n    plt.imshow(image)\n\n    # remove the axis / ticks for a clean looking image\n    plt.xticks([])\n    plt.yticks([])\n\n    # if a title is provided, show it\n    if title is not None:\n        plt.title(title)\n\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nimg_folder = '/kaggle/working/train/train/5.png'\nimage = cv2.imread(img_folder)\nshow_rgb_image(image, 'Original Image')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Helper functions\nclass CIFAR10Dataset(Dataset):\n    def __init__(self, img_dir, labels_df=None, transform=None, is_test=False):\n        self.img_dir = img_dir\n        self.labels_df = labels_df\n        self.transform = transform\n        self.is_test = is_test\n\n    def __len__(self):\n        return len(self.labels_df) if not self.is_test else 300000  # Adjust for test size\n\n    def __getitem__(self, idx):\n        if self.is_test:\n            img_path = os.path.join(self.img_dir, f\"{idx+1}.png\")\n            image = Image.open(img_path)\n            if self.transform:\n                image = self.transform(image)\n            return image\n        else:\n            img_path = os.path.join(self.img_dir, f\"{idx+1}.png\")\n            label = self.labels_df.iloc[idx, 1]\n            label = class_to_idx[label]\n            image = Image.open(img_path)\n            if self.transform:\n                image = self.transform(image)\n            return image, label\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load training labels\ntrain_labels_df = pd.read_csv(labels_csv)\nclasses = train_labels_df['label'].unique().tolist()\nclass_to_idx = {cls: idx for idx, cls in enumerate(classes)}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the labels from the CSV file ## NEW\n#labels_df = pd.read_csv(labels_csv)\n#image_labels = {str(row['id']): row['label'] for _, row in labels_df.iterrows()}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\n\n# Cutout implementation\nclass Cutout(object):\n    def __init__(self, n_holes, length):\n        self.n_holes = n_holes\n        self.length = length\n\n    def __call__(self, img):\n        h, w = img.size(1), img.size(2)\n        mask = np.ones((h, w), np.float32)\n\n        for n in range(self.n_holes):\n            y = np.random.randint(h)\n            x = np.random.randint(w)\n\n            y1 = np.clip(y - self.length // 2, 0, h)\n            y2 = np.clip(y + self.length // 2, 0, h)\n            x1 = np.clip(x - self.length // 2, 0, w)\n            x2 = np.clip(x + self.length // 2, 0, w)\n\n            mask[y1: y2, x1: x2] = 0.\n\n        mask = torch.from_numpy(mask)\n        mask = mask.expand_as(img)\n        img = img * mask\n\n        return img\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data augmentations with CutOut\ntrain_transforms = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomRotation(15),  # New augmentation\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2),  # New augmentation\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010]),\n    Cutout(n_holes=1, length=8),\n])\n\ntest_transforms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010]),\n])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function for CutMix augmentation\ndef cutmix(data, targets, alpha=1.0):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n\n    lam = np.random.beta(alpha, alpha)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    data[:, :, bbx1:bbx2, bby1:bby2] = shuffled_data[:, :, bbx1:bbx2, bby1:bby2]\n\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    return data, targets, shuffled_targets, lam\n\n\n# Random bounding box for CutMix\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Datasets and Dataloaders\ntrain_dataset = CIFAR10Dataset(train_dir, train_labels_df, transform=train_transforms)\ntrain_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n\ntest_dataset = CIFAR10Dataset(test_dir, transform=test_transforms, is_test=True)\ntest_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model (ResNet without pre-training)\nmodel = models.resnet18(weights=None)\nmodel.fc = nn.Linear(model.fc.in_features, num_classes)  # Adjust for CIFAR-10\nmodel = model.to(device)\n\n# Model (ResNet without pre-training)\n#model = models.resnet34(weights=None)  # Switching to a deeper ResNet\n#model.fc = nn.Linear(model.fc.in_features, num_classes)\n#model = model.to(device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Loss function and optimizer\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1)\noptimizer = torch.optim.AdamW(model.parameters(), lr=0.001, weight_decay=1e-4)\n#optimizer = optim.AdamW(model.parameters(), lr=learning_rate)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3: Define learning rate scheduler (CosineAnnealing with Warmup)\n# Warmup for the first 5 epochs, then use cosine annealing\n         \nclass WarmupCosineAnnealingLR(torch.optim.lr_scheduler._LRScheduler):\n    def __init__(self, optimizer, warmup_epochs, max_epochs, eta_min=1e-5, last_epoch=-1):\n        self.warmup_epochs = warmup_epochs\n        self.max_epochs = max_epochs\n        self.eta_min = eta_min\n        self.cosine_scheduler = CosineAnnealingLR(optimizer, T_max=max_epochs - warmup_epochs, eta_min=eta_min)\n        super(WarmupCosineAnnealingLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        if self.last_epoch < self.warmup_epochs:\n            # Linear warmup\n            warmup_lr = [base_lr * (self.last_epoch + 1) / self.warmup_epochs for base_lr in self.base_lrs]\n            return warmup_lr\n        else:\n            # Use cosine annealing after warmup\n            return self.cosine_scheduler.get_last_lr()\n\n    def step(self, epoch=None):\n        if self.last_epoch < self.warmup_epochs:\n            super(WarmupCosineAnnealingLR, self).step(epoch)\n        else:\n            # Step without passing `epoch` to avoid deprecation warning\n            self.cosine_scheduler.step()  # Just step without epoch\n        self.last_epoch += 1  # Ensure last_epoch is incremented manually\n\n# Define the warmup + cosine annealing schedule\nscheduler = WarmupCosineAnnealingLR(optimizer, warmup_epochs=20, max_epochs=500)\n#scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=2)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cosine Annealing LR with Warmup\n#def cosine_annealing_with_warmup(epoch):\n#    if epoch < warmup_epochs:\n#        return epoch / warmup_epochs\n#    else:\n#        return 0.5 * (1 + np.cos((epoch - warmup_epochs) / (num_epochs - warmup_epochs) * np.pi))\n\n\n###scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=100, eta_min=1e-6)\n#scheduler = LambdaLR(optimizer, lr_lambda=cosine_annealing_with_warmup)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training function with CutMix\ndef train(model, train_loader, criterion, optimizer, epoch, alpha=1.0, cutmix_prob=0.5):\n    model.train()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    for images, labels in tqdm(train_loader):\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n\n        # Apply CutMix with a probability\n        if random.random() < cutmix_prob:\n            images, targets_a, targets_b, lam = cutmix(images, labels, alpha)\n            outputs = model(images)\n            loss = lam * criterion(outputs, targets_a) + (1 - lam) * criterion(outputs, targets_b)\n        else:\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n\n        loss.backward() # Backward pass\n        \n        # Add gradient clipping here\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        \n        optimizer.step() # Update weights\n\n        running_loss += loss.item()\n        _, predicted = torch.max(outputs, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\n    accuracy = 100.0 * correct / total\n    print(f\"Epoch {epoch}, Loss: {running_loss/len(train_loader)}, Accuracy: {accuracy}%\")\n    return running_loss/len(train_loader), accuracy\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Testing function\ndef test(model, test_loader):\n    model.eval()\n    all_preds = []\n    with torch.no_grad():\n        for images in tqdm(test_loader):\n            images = images.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            all_preds.extend(predicted.cpu().numpy())\n    return all_preds","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training Loop\n#for epoch in range(num_epochs):\n#    train_loss, train_acc = train(model, train_loader, criterion, optimizer, epoch)\n#    scheduler.step()\n    \nfor epoch in range(num_epochs):\n    train_loss, train_acc = train(model, train_loader, criterion, optimizer, epoch)\n    scheduler.step()  \n    # Print current learning rate\n    # print(f\"Epoch {epoch}, Current Learning Rate: {scheduler.get_last_lr()[0]}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the model\ntorch.save(model.state_dict(), \"cifar10_resnet18_3.pth\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Testing on 300k test images\ntest_preds = test(model, test_loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare submission dataframe\nimage_indices = list(range(1, len(test_preds) + 1))  # Image indices from 1 to 300000\npredicted_labels = [classes[pred] for pred in test_preds]  # Map predicted indices to class names\n\n# Create a DataFrame for submission\nsubmission_df = pd.DataFrame({\n    'id': image_indices,   # Image index\n    'label': predicted_labels  # Predicted label in text form\n})\n\n# Save submission to a CSV file\nsubmission_df.to_csv('submission_2.csv', index=False)\n\nprint(\"Submission file 'submission.csv' created successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}