{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Advanced CIFAR-10 Classification Challenge in DSEG660 by Ali Aldhubri\n#### I employs cutting-edge techniques to train a neural network from scratch, aiming to achieve top scores in CIFAR-10 image classification on Kaggle, showcasing skills in advanced optimization, learning rate scheduling, and data augmentation.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.datasets as datasets\nfrom torchvision.models import resnet101\nfrom torchvision import transforms\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport torchinfo\nimport random\nimport os\n\n# Set seed\nrandom_seed = 42\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\nos.environ[\"PYTHONHASHSEED\"] = str(random_seed)\ntorch.manual_seed(random_seed)\ntorch.cuda.manual_seed(random_seed)\ntorch.cuda.manual_seed_all(random_seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.use_deterministic_algorithms(False)\n\n# Hyperparameters\nbatch_size = 128  # Increased for better stability\nlearning_rate = 1e-3  # Higher starting learning rate\nnum_epochs = 100\nimg_size = 128\nwarmup_epochs = 5\nweight_decay = 1e-3  # Increased weight decay\n\n# Custom CutOut augmentation\nclass CutOut(object):\n    def __init__(self, length):\n        self.length = length\n\n    def __call__(self, img):\n        h, w = img.shape[1], img.shape[2]\n        mask = np.ones((h, w), np.float32)\n        y = np.random.randint(h)\n        x = np.random.randint(w)\n        y1 = np.clip(y - self.length // 2, 0, h)\n        y2 = np.clip(y + self.length // 2, 0, h)\n        x1 = np.clip(x - self.length // 2, 0, w)\n        x2 = np.clip(x + self.length // 2, 0, w)\n        mask[y1: y2, x1: x2] = 0.\n        img = img * torch.tensor(mask, dtype=torch.float32).unsqueeze(0)\n        return img\n\n# Changed function name from cutmix to mixed_cut\ndef mixed_cut(data, targets, alpha=1.0):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    lam = np.clip(np.random.beta(alpha, alpha), 0.3, 0.7)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    data[:, :, bbx1:bbx2, bby1:bby2] = shuffled_data[:, :, bbx1:bbx2, bby1:bby2]\n    return data, targets, shuffled_targets, lam\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\n# Define transforms\ntrain_transforms = transforms.Compose([\n    transforms.Resize(img_size),\n    transforms.RandomRotation(20),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.ToTensor(),\n    CutOut(length=16),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ntest_transforms = transforms.Compose([\n    transforms.Resize(img_size),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Load CIFAR10 dataset\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, transform=train_transforms, download=True)\nvalid_dataset = datasets.CIFAR10(root='./data', train=False, transform=test_transforms, download=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n\n# Squeeze-and-Excitation (SE) Block\nclass SEBlock(nn.Module):\n    def __init__(self, channel, reduction=16):\n        super(SEBlock, self).__init__()\n        self.fc1 = nn.Linear(channel, channel // reduction, bias=False)\n        self.fc2 = nn.Linear(channel // reduction, channel, bias=False)\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = torch.mean(x, dim=(2, 3), keepdim=True)\n        y = y.view(b, c)\n        y = torch.relu(self.fc1(y))\n        y = torch.sigmoid(self.fc2(y)).view(b, c, 1, 1)\n        return x * y\n\n# Define the model\nclass ComplexResNet101(nn.Module):\n    def __init__(self, num_classes=10):\n        super(ComplexResNet101, self).__init__()\n        self.model = resnet101(weights=None)\n        self.in_features = self.model.fc.in_features\n        self.model.fc = nn.Sequential(\n            nn.Dropout(0.5),\n            nn.Linear(self.in_features, 512),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, num_classes)\n        )\n        self.se_block = SEBlock(self.in_features)\n\n    def forward(self, x):\n        x = self.model.conv1(x)\n        x = self.model.bn1(x)\n        x = self.model.relu(x)\n        x = self.model.maxpool(x)\n\n        x = self.model.layer1(x)\n        x = self.model.layer2(x)\n        x = self.model.layer3(x)\n        x = self.se_block(self.model.layer4(x))\n\n        x = self.model.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.model.fc(x)\n        return x\n\n# Instantiate the model and optimizer\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nmodel = ComplexResNet101(num_classes=10).to(device)\noptimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n\n# Learning rate scheduler with warmup and cosine annealing\nscheduler_steplr = torch.optim.lr_scheduler.LambdaLR(\n    optimizer,\n    lr_lambda=lambda epoch: epoch / warmup_epochs if epoch < warmup_epochs else 1.\n)\nscheduler_cosine = CosineAnnealingLR(optimizer, T_max=num_epochs - warmup_epochs)\n\n# Loss function with label smoothing\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1).to(device)\n\n# Training loop with mixed_cut and gradient clipping\nbest_epoch = 0\nbest_acc = 0.0\nfor epoch in range(1, num_epochs + 1):\n    print(f\"Epoch {epoch}/{num_epochs}\")\n\n    model.train()\n    epoch_loss = 0.0\n    epoch_corrects = 0\n    for batch_in, batch_out in tqdm(train_loader):\n        batch_in, batch_out = batch_in.to(device), batch_out.to(device)\n\n        # Use both CutOut and mixed_cut simultaneously for stronger augmentation\n        if np.random.rand(1) < 0.5:\n            batch_in, targets, shuffled_targets, lam = mixed_cut(batch_in, batch_out)\n            preds = model(batch_in)\n            loss = lam * criterion(preds, targets) + (1 - lam) * criterion(preds, shuffled_targets)\n        else:\n            preds = model(batch_in)\n            loss = criterion(preds, batch_out)\n\n        optimizer.zero_grad()\n        loss.backward()\n\n        # Gradient clipping to prevent exploding gradients\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n\n        optimizer.step()\n\n        epoch_loss += loss.item() * batch_in.size(0)\n        epoch_corrects += torch.sum(torch.argmax(preds, 1) == batch_out)\n\n    epoch_loss /= len(train_loader.dataset)\n    epoch_acc = epoch_corrects.float() / len(train_loader.dataset)\n    print(f\"Train loss: {epoch_loss:.4f}, Train acc: {epoch_acc:.4f}\")\n\n    # Validation\n    model.eval()\n    with torch.no_grad():\n        epoch_loss = 0.0\n        epoch_corrects = 0\n        for batch_in, batch_out in tqdm(valid_loader):\n            batch_in, batch_out = batch_in.to(device), batch_out.to(device)\n            preds = model(batch_in)\n            loss = criterion(preds, batch_out)\n            epoch_loss += loss.item() * batch_in.size(0)\n            epoch_corrects += torch.sum(torch.argmax(preds, 1) == batch_out)\n\n        epoch_loss /= len(valid_loader.dataset)\n        epoch_acc = epoch_corrects.float() / len(valid_loader.dataset)\n\n    print(f\"Valid loss: {epoch_loss:.4f}, Valid acc: {epoch_acc:.4f}\")\n\n    # Update best model\n    if epoch_acc > best_acc:\n        best_acc = epoch_acc\n        best_epoch = epoch\n        torch.save(model.state_dict(), 'best_model.pth')\n\n    # Step the schedulers\n    scheduler_steplr.step() if epoch <= warmup_epochs else scheduler_cosine.step()\n\n    print(f\"Best epoch: {best_epoch}, Best acc: {best_acc:.4f}\")\n\n# Print the best accuracy at the end of training\nprint(f\"Training complete! Best accuracy: {best_acc:.4f} at epoch {best_epoch}\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-09T06:50:50.237961Z","iopub.execute_input":"2024-10-09T06:50:50.23887Z","iopub.status.idle":"2024-10-09T16:40:22.454383Z","shell.execute_reply.started":"2024-10-09T06:50:50.238825Z","shell.execute_reply":"2024-10-09T16:40:22.453369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nlearning_rate = 1e-4\nnum_epochs = 200\nimg_size = 128","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:54:37.609923Z","iopub.execute_input":"2024-10-09T16:54:37.610229Z","iopub.status.idle":"2024-10-09T16:54:37.614389Z","shell.execute_reply.started":"2024-10-09T16:54:37.610192Z","shell.execute_reply":"2024-10-09T16:54:37.613597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read csv\nimport pandas as pd\nsample = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:54:37.61529Z","iopub.execute_input":"2024-10-09T16:54:37.615579Z","iopub.status.idle":"2024-10-09T16:54:37.705168Z","shell.execute_reply.started":"2024-10-09T16:54:37.615549Z","shell.execute_reply":"2024-10-09T16:54:37.704254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the form of data\nsample.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:54:37.706323Z","iopub.execute_input":"2024-10-09T16:54:37.706649Z","iopub.status.idle":"2024-10-09T16:54:37.716459Z","shell.execute_reply.started":"2024-10-09T16:54:37.706616Z","shell.execute_reply":"2024-10-09T16:54:37.715526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decompression data\n!pip install py7zr\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:54:37.717655Z","iopub.execute_input":"2024-10-09T16:54:37.717939Z","iopub.status.idle":"2024-10-09T16:57:19.949334Z","shell.execute_reply.started":"2024-10-09T16:54:37.717908Z","shell.execute_reply":"2024-10-09T16:57:19.948419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a list of data filenames for dataset\nimport os\ntest_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:57:19.950869Z","iopub.execute_input":"2024-10-09T16:57:19.951391Z","iopub.status.idle":"2024-10-09T16:57:20.971478Z","shell.execute_reply.started":"2024-10-09T16:57:19.951325Z","shell.execute_reply":"2024-10-09T16:57:20.970651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n\nclass CIFAR10(Dataset):\n    def __init__(self, filenames, transforms):\n        self.filenames = filenames\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx])\n        transformed_img = self.transforms(img)\n        return transformed_img","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:57:20.972771Z","iopub.execute_input":"2024-10-09T16:57:20.973148Z","iopub.status.idle":"2024-10-09T16:57:20.979767Z","shell.execute_reply.started":"2024-10-09T16:57:20.973108Z","shell.execute_reply":"2024-10-09T16:57:20.978847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load dataset\ntest_dataset = CIFAR10(test_filenames, test_transforms)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:57:20.980917Z","iopub.execute_input":"2024-10-09T16:57:20.981218Z","iopub.status.idle":"2024-10-09T16:57:21.021026Z","shell.execute_reply.started":"2024-10-09T16:57:20.98118Z","shell.execute_reply":"2024-10-09T16:57:21.02019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For one-hot encoding\nclasses = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:57:21.022113Z","iopub.execute_input":"2024-10-09T16:57:21.022436Z","iopub.status.idle":"2024-10-09T16:57:21.031233Z","shell.execute_reply.started":"2024-10-09T16:57:21.022402Z","shell.execute_reply":"2024-10-09T16:57:21.030407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the best model\nimport torch\nfrom PIL import Image\n# Get the result\nresult = []\n\nwith torch.no_grad():\n    model.eval()\n    for batch_in in tqdm(test_loader):\n        batch_in = batch_in.to(device)\n        \n        y_pred = model(batch_in)\n        preds = torch.argmax(y_pred, 1)\n        preds = preds.cpu().numpy()\n        result.extend([classes[i] for i in preds])\n\nsample['label'] = result\nsample.to_csv('./submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T16:57:21.03235Z","iopub.execute_input":"2024-10-09T16:57:21.03277Z","iopub.status.idle":"2024-10-09T17:08:57.721994Z","shell.execute_reply.started":"2024-10-09T16:57:21.032727Z","shell.execute_reply":"2024-10-09T17:08:57.721127Z"},"trusted":true},"execution_count":null,"outputs":[]}]}