{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries and Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install py7zr -q","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-01T12:42:05.514612Z","iopub.execute_input":"2024-10-01T12:42:05.514967Z","iopub.status.idle":"2024-10-01T12:42:20.600989Z","shell.execute_reply.started":"2024-10-01T12:42:05.514932Z","shell.execute_reply":"2024-10-01T12:42:20.599784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom typing import List, Optional\n\nimport os\nimport pickle\nfrom py7zr import SevenZipFile\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom PIL import Image\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, Dataset, Subset\nfrom torchvision import transforms\nfrom torchvision.datasets import ImageFolder\nfrom torchvision.models import alexnet, resnet50, AlexNet_Weights, ResNet50_Weights\ntorch.manual_seed(42)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-01T15:18:28.808767Z","iopub.execute_input":"2024-10-01T15:18:28.809602Z","iopub.status.idle":"2024-10-01T15:18:28.819626Z","shell.execute_reply.started":"2024-10-01T15:18:28.809550Z","shell.execute_reply":"2024-10-01T15:18:28.818592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{"execution":{"iopub.status.busy":"2024-09-25T13:40:32.933177Z","iopub.execute_input":"2024-09-25T13:40:32.933620Z","iopub.status.idle":"2024-09-25T13:40:32.938054Z","shell.execute_reply.started":"2024-09-25T13:40:32.933580Z","shell.execute_reply":"2024-09-25T13:40:32.937168Z"}}},{"cell_type":"code","source":"train_labels_path = '/kaggle/input/cifar-10/trainLabels.csv'\ntrain_archive_path = '/kaggle/input/cifar-10/train.7z'\ntrain_extract_path = \"/kaggle/working/train_images\"\n\ndef extract_7z_archive(archive_path, extract_path):\n    with SevenZipFile(archive_path, mode='r') as z:\n        z.extractall(path=extract_path)\n\nextract_7z_archive(train_archive_path, train_extract_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T12:43:57.082987Z","iopub.execute_input":"2024-10-01T12:43:57.083872Z","iopub.status.idle":"2024-10-01T12:44:22.104217Z","shell.execute_reply.started":"2024-10-01T12:43:57.083831Z","shell.execute_reply":"2024-10-01T12:44:22.103124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Transformation","metadata":{}},{"cell_type":"code","source":"# IMAGE_SIZE = 32\nIMAGE_SIZE = 224 # a must for alexnet, below 224 will be too small for alexnet\nmean, std = [.4914, .4822, .4465], [.247, .243, .261] # ppl discoveries for fast convergence\n\ncomposed_train = transforms.Compose([\n    transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    transforms.RandomRotation(20),\n    transforms.RandomHorizontalFlip(.1),\n    transforms.ColorJitter(brightness=.1, contrast=.1, saturation=.1),\n    transforms.RandomAdjustSharpness(sharpness_factor=2, p=.1),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std),\n    transforms.RandomErasing(p=.75, scale=(.02, .1), value=1., inplace=False)\n])\n\ncomposed_test = transforms.Compose([\n    transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std),\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-01T13:52:44.909016Z","iopub.execute_input":"2024-10-01T13:52:44.909414Z","iopub.status.idle":"2024-10-01T13:52:44.917446Z","shell.execute_reply.started":"2024-10-01T13:52:44.909377Z","shell.execute_reply":"2024-10-01T13:52:44.916409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ImageFolder for training and testing\nThe standard `ImageFolder` class cannot categorize images based on labels from a CSV file since our training images are not organized into separate folders. To address this, we implement a `CustomImageFolder` that overrides the default behavior. This custom class allows us to manage custom labels and ensures that the `__getitem__` method returns the correct image-label pairs, enabling more straightfoward iteration over the dataset.","metadata":{}},{"cell_type":"code","source":"class CustomImageFolder(ImageFolder):\n    def __init__(self, root, custom_labels: Optional[List[str]] = None, **ImageFolderKwargs):\n        super().__init__(root, **ImageFolderKwargs)\n        self.custom_labels = None\n        if isinstance(custom_labels, list):\n            self.set_custom_labels(custom_labels)\n            \n    def set_custom_labels(self, custom_labels: List[str]):\n        self.custom_labels = custom_labels\n        self.classes = sorted(set(custom_labels))\n        self.class_to_idx = {_c: n for n, _c in enumerate(self.classes)}\n    \n    def __getitem__(self, index):\n        image, label = super().__getitem__(index)\n        if isinstance(self.custom_labels, list):\n            label_str = self.custom_labels[index]\n            label = torch.tensor(self.classes.index(label_str), dtype=torch.long)\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:10:38.544136Z","iopub.execute_input":"2024-10-01T15:10:38.544541Z","iopub.status.idle":"2024-10-01T15:10:38.552469Z","shell.execute_reply.started":"2024-10-01T15:10:38.544503Z","shell.execute_reply":"2024-10-01T15:10:38.551335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = CustomImageFolder(train_extract_path)\n\nlabels_df = pd.read_csv(train_labels_path)\nimage_ids = [int(p.split('/')[-1].replace('.png', '')) for p, _ in dataset.imgs]\nlabels_list = [labels_df[labels_df['id'] == img_id].label.values[0] for img_id in image_ids]\n\ndataset.imgs = list(zip([p for p, _ in dataset.imgs], labels_list))\ndataset.targets = labels_list\ndataset.set_custom_labels(labels_list)\n\ntrain_idx, test_idx = train_test_split(list(range(len(dataset))), test_size=.2, stratify=dataset.targets, random_state=42)\nval_idx, test_idx = train_test_split(test_idx, test_size=.5, stratify=[dataset.targets[i] for i in test_idx], random_state=42)\n\ntrain_dataset = Subset(dataset, train_idx)\nval_dataset = Subset(dataset, val_idx)\ntest_dataset = Subset(dataset, test_idx)\n\ntrain_dataset.dataset.transform = composed_train\nval_dataset.dataset.transform = composed_test\ntest_dataset.dataset.transform = composed_test\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:10:38.840979Z","iopub.execute_input":"2024-10-01T15:10:38.841325Z","iopub.status.idle":"2024-10-01T15:10:57.832350Z","shell.execute_reply.started":"2024-10-01T15:10:38.841290Z","shell.execute_reply":"2024-10-01T15:10:57.831523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Device Assignment\nThe `DeviceDataLoader` class is designed to simplify the process of moving batches of data from a `DataLoader` to a specified device (e.g., GPU). By overriding the `__iter__` method, it ensures that each batch is automatically transferred to the desired device using the `to_device` function, which handles both individual tensors and collections of data. ","metadata":{}},{"cell_type":"code","source":"def to_device(data, device: torch.device):\n    if isinstance(data, (list,tuple)):\n        return [to_device(x, device) for x in data]\n    return data.to(device, non_blocking=True)\n\nclass DeviceDataLoader():\n    def __init__(self, dataloader: DataLoader, device: torch.device):\n        self.dl: DataLoader = dataloader\n        self.device: torch.device = device\n        \n    def __iter__(self):\n        for b in self.dl: \n            yield to_device(b, self.device)\n\n    def __len__(self):\n        return len(self.dl)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:10:57.834132Z","iopub.execute_input":"2024-10-01T15:10:57.834843Z","iopub.status.idle":"2024-10-01T15:10:57.841658Z","shell.execute_reply.started":"2024-10-01T15:10:57.834798Z","shell.execute_reply":"2024-10-01T15:10:57.840506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Integration of DeviceDataLoader","metadata":{}},{"cell_type":"code","source":"train_ddl = DeviceDataLoader(train_loader, device)\nval_ddl = DeviceDataLoader(val_loader, device)\ntest_ddl = DeviceDataLoader(test_loader, device)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:11:04.555570Z","iopub.execute_input":"2024-10-01T15:11:04.556342Z","iopub.status.idle":"2024-10-01T15:11:04.560871Z","shell.execute_reply.started":"2024-10-01T15:11:04.556301Z","shell.execute_reply":"2024-10-01T15:11:04.559903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DCNN Architecture","metadata":{}},{"cell_type":"code","source":"class DCNN_:\n    def __init__(self, out_features: int, device: Optional[torch.device] = None):\n        self.out_features = out_features\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if isinstance(device, torch.device) else device\n    \n    def get_alexnet(self):\n        model = alexnet(weights=AlexNet_Weights.DEFAULT)\n        for param in model.features.parameters():\n            param.requires_grad = False\n        model.classifier[-1] = nn.Linear(model.classifier[-1].in_features, self.out_features)\n        nn.init.xavier_uniform_(model.classifier[-1].weight)\n        model = model.to(self.device)\n        return model\n\n    def get_resnet50(self):\n        model = resnet50(weights=ResNet50_Weights.DEFAULT)\n        for param in model.parameters():\n            param.requires_grad = False\n        model.fc = nn.Linear(model.fc.in_features, self.out_features)\n        nn.init.xavier_uniform_(model.fc.weight)\n        model = model.to(self.device)\n        return model\n    \ndcnn = DCNN_(out_features=len(dataset.classes), device=device)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:11:06.168604Z","iopub.execute_input":"2024-10-01T15:11:06.169519Z","iopub.status.idle":"2024-10-01T15:11:06.179668Z","shell.execute_reply.started":"2024-10-01T15:11:06.169476Z","shell.execute_reply":"2024-10-01T15:11:06.178632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training & Validation","metadata":{}},{"cell_type":"code","source":"def train(model, train_ddl: DeviceDataLoader, val_ddl: DeviceDataLoader, criterion, optimizer, device: torch.device, epochs: int = 10, checkpoint_epochs: int = 10):\n    history = {'train': {'acc': [], 'loss': []}, 'val': {'acc': [], 'loss': []}}\n    \n    start = time.time()\n    print(f'Training for {epochs} epochs on {device}')\n    \n    for epoch in range(1, epochs+1):\n        print(f\"Epoch {epoch}/{epochs}\")\n        \n        # Training\n        model.train()\n        train_loss = torch.tensor(0., device=device)\n        train_acc = torch.tensor(0., device=device)\n        for X, y in train_ddl:\n            preds = model(X)\n            loss = criterion(preds, y)\n            \n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            \n            with torch.no_grad():\n                train_loss += loss * train_ddl.dl.batch_size\n                train_acc += (torch.argmax(preds, dim=1) == y).sum()\n        \n        # Validation\n        model.eval()  # put network in train mode for Dropout and Batch Normalization\n        val_loss = torch.tensor(0., device=device)\n        val_acc = torch.tensor(0., device=device)\n        with torch.no_grad():\n            for X, y in val_ddl:\n                preds = model(X)\n                loss = criterion(preds, y)\n\n                val_loss += loss * val_ddl.dl.batch_size\n                val_acc += (torch.argmax(preds, dim=1) == y).sum()\n        \n        # Evaluation\n        overall_train_loss = train_loss / len(train_ddl.dl.dataset)\n        overall_train_acc = 100 * train_acc / len(train_ddl.dl.dataset)\n        history['train']['loss'].append(overall_train_loss.cpu())\n        history['train']['acc'].append(overall_train_acc.cpu())\n        \n        overall_val_loss = val_loss / len(val_ddl.dl.dataset)\n        overall_val_acc = 100 * val_acc / len(val_ddl.dl.dataset)\n        history['val']['loss'].append(overall_val_loss.cpu())\n        history['val']['acc'].append(overall_val_acc.cpu())\n        \n        print(f'Training loss: {overall_train_loss:.5f}')\n        print(f'Training accuracy: {overall_train_acc:.2f}')\n\n        print(f'Val loss: {overall_val_loss:.5f}')\n        print(f'Val accuracy: {overall_val_acc:.2f}')\n        \n        if epoch % checkpoint_epochs == 0:\n            torch.save({\n                'epoch': epoch,\n                'state_dict': model.state_dict(),\n                'optimizer': optimizer.state_dict(),\n            }, './checkpoint.pth.tar')\n        \n        print()\n    \n    end = time.time()\n    print(f'Total training time: {end - start:.2f} seconds')\n    return history","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:11:07.491857Z","iopub.execute_input":"2024-10-01T15:11:07.492240Z","iopub.status.idle":"2024-10-01T15:11:07.507707Z","shell.execute_reply.started":"2024-10-01T15:11:07.492202Z","shell.execute_reply":"2024-10-01T15:11:07.506601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation Helpers","metadata":{}},{"cell_type":"code","source":"def show_model_lr(history):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\n\n    # Plotting Accuracy\n    ax1.plot(history['train']['acc'], label='Train Accuracy', marker='o')\n    ax1.plot(history['val']['acc'], label='Validation Accuracy', marker='o')\n    ax1.set_title('Training and Validation Accuracy')\n    ax1.set_xlabel('Epochs')\n    ax1.set_ylabel('Accuracy')\n    ax1.legend()\n    ax1.grid()\n\n    # Plotting Loss\n    ax2.plot(history['train']['loss'], label='Train Loss', marker='o')\n    ax2.plot(history['val']['loss'], label='Validation Loss', marker='o')\n    ax2.set_title('Training and Validation Loss')\n    ax2.set_xlabel('Epochs')\n    ax2.set_ylabel('Loss')\n    ax2.legend()\n    ax2.grid()\n\n    # Adjust layout\n    plt.tight_layout()\n    plt.show()\n\n\ndef get_preds(model, ddl, device: torch.device):\n    y_pred = []\n    y_true = []\n\n    model.eval()\n    with torch.no_grad():\n        for X, y in ddl:\n            X = X.to(device)\n            y_pred.extend(model(X).argmax(dim=1).type(torch.int32).cpu().numpy())\n            y_true.extend(y.type(torch.int32).cpu().numpy())\n    return y_pred, y_true\n\n\ndef show_confusion_matrix(y_true, y_pred, classes_: List[str]):\n    plt.figure(figsize=(15, 9))\n    cm = confusion_matrix(y_true, y_pred)\n\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=classes_)\n    disp.plot(cmap=plt.cm.Blues)\n\n    plt.title('Confusion Matrix')\n    plt.xticks(rotation=45)\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"## AlexNet","metadata":{}},{"cell_type":"code","source":"# epochs = 15\n# lr = 1e-5\n# weight_decay = 5e-4\n\n# alexnet_model = dcnn.get_alexnet()\n# criterion = nn.CrossEntropyLoss()\n# optimizer = optim.Adam(alexnet_model.classifier[-1].parameters(), lr=lr, weight_decay=weight_decay)\n\n# hist = train(\n#     alexnet_model, \n#     train_ddl=train_ddl, \n#     val_ddl=val_ddl, \n#     criterion=criterion, \n#     optimizer=optimizer, \n#     device=device,\n#     epochs=epochs,\n# )\n# show_model_lr(hist)\n# y_pred, y_true = get_preds(alexnet_model, test_ddl, device)\n# print(classification_report(y_true, y_pred))\n# show_confusion_matrix(y_true, y_pred, classes_=dataset.classes)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-10-01T15:11:09.847006Z","iopub.execute_input":"2024-10-01T15:11:09.847651Z","iopub.status.idle":"2024-10-01T15:14:46.971076Z","shell.execute_reply.started":"2024-10-01T15:11:09.847606Z","shell.execute_reply":"2024-10-01T15:14:46.970036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet50","metadata":{}},{"cell_type":"code","source":"epochs = 20\nlr = 1e-5\nweight_decay = 5e-4\n\nresnet50_model = dcnn.get_resnet50()\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(resnet50_model.fc.parameters(), lr=lr, weight_decay=weight_decay)\n\nhist = train(\n    resnet50_model, \n    train_ddl=train_ddl, \n    val_ddl=val_ddl, \n    criterion=criterion, \n    optimizer=optimizer, \n    device=device,\n    epochs=epochs,\n)\nshow_model_lr(hist)\ny_pred, y_true = get_preds(resnet50_model, test_ddl, device)\nprint(classification_report(y_true, y_pred))\nshow_confusion_matrix(y_true, y_pred, classes_=dataset.classes)","metadata":{"execution":{"iopub.status.busy":"2024-10-01T13:50:52.265860Z","iopub.execute_input":"2024-10-01T13:50:52.266238Z","iopub.status.idle":"2024-10-01T13:50:52.270659Z","shell.execute_reply.started":"2024-10-01T13:50:52.266204Z","shell.execute_reply":"2024-10-01T13:50:52.269716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-10-01T15:18:09.059892Z","iopub.execute_input":"2024-10-01T15:18:09.060759Z","iopub.status.idle":"2024-10-01T15:18:17.328051Z","shell.execute_reply.started":"2024-10-01T15:18:09.060719Z","shell.execute_reply":"2024-10-01T15:18:17.327133Z"},"trusted":true},"execution_count":null,"outputs":[]}]}