{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Visual Transformer with Pre-trained ViT-B/16\n\n## Training Visual Transformer on *Dogs vs Cats Data*\n\n* Dogs vs Cats Redux: Kernels Edition - https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition\n* Base Code - https://www.kaggle.com/reukki/pytorch-cnn-tutorial-with-cats-and-dogs/n\n","metadata":{}},{"cell_type":"markdown","source":"## Imports and Setup","metadata":{}},{"cell_type":"code","source":"!pip install torchvision","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:06.814799Z","iopub.execute_input":"2024-12-22T12:14:06.815056Z","iopub.status.idle":"2024-12-22T12:14:16.361980Z","shell.execute_reply.started":"2024-12-22T12:14:06.815029Z","shell.execute_reply":"2024-12-22T12:14:16.361122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from __future__ import print_function\n\nimport glob\nimport os\nimport random\nimport zipfile\nimport shutil\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms, models\nfrom tqdm.notebook import tqdm\n\nprint(f\"Torch: {torch.__version__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:16.364834Z","iopub.execute_input":"2024-12-22T12:14:16.365535Z","iopub.status.idle":"2024-12-22T12:14:20.985489Z","shell.execute_reply.started":"2024-12-22T12:14:16.365481Z","shell.execute_reply":"2024-12-22T12:14:20.984634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training parameters\nbatch_size = 64\nepochs = 10\nlr = 3e-5\ngamma = 0.7\nseed = 42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:20.986903Z","iopub.execute_input":"2024-12-22T12:14:20.987254Z","iopub.status.idle":"2024-12-22T12:14:20.990700Z","shell.execute_reply.started":"2024-12-22T12:14:20.987226Z","shell.execute_reply":"2024-12-22T12:14:20.989921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(seed)\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:20.991883Z","iopub.execute_input":"2024-12-22T12:14:20.992221Z","iopub.status.idle":"2024-12-22T12:14:21.080768Z","shell.execute_reply.started":"2024-12-22T12:14:20.992183Z","shell.execute_reply":"2024-12-22T12:14:21.079700Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Loading","metadata":{}},{"cell_type":"code","source":"os.makedirs('data', exist_ok=True)\ntrain_dir = 'data/train'\ntest_dir = 'data/test'\n\n# Extract zip files\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('data')\n\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('data')\n\n# List all training and test files\ntrain_list = glob.glob(os.path.join(train_dir, '*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\n\n# We will split the training data into our own train/validation sets\ntrain_split_dir = 'data/train_split'\ntest_split_dir = 'data/test_split'\nos.makedirs(train_split_dir, exist_ok=True)\nos.makedirs(test_split_dir, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:21.081896Z","iopub.execute_input":"2024-12-22T12:14:21.082130Z","iopub.status.idle":"2024-12-22T12:14:34.353279Z","shell.execute_reply.started":"2024-12-22T12:14:21.082107Z","shell.execute_reply":"2024-12-22T12:14:34.352306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gather file paths and labels\nfile_paths = glob.glob(os.path.join(train_dir, '*.jpg'))\nlabels = [1 if 'dog' in os.path.basename(path) else 0 for path in file_paths]\n\n# Split into train/test_split (80% train, 20% validation for later usage)\ntrain_list, test_split_list, train_labels, test_split_labels = train_test_split(\n    file_paths, labels, test_size=0.2, stratify=labels, random_state=42\n)\n\n# Copy files into respective directories\nfor file_path in train_list:\n    shutil.copy(file_path, train_split_dir)\nfor file_path in test_split_list:\n    shutil.copy(file_path, test_split_dir)\n\nprint(f\"Train Split: {len(train_list)} images\")\nprint(f\"Validation Split: {len(test_split_list)} images\")\n\n# For random display (labels here are just for the random sample)\nlabels_str = [os.path.basename(path).split('.')[0] for path in train_list]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:34.354473Z","iopub.execute_input":"2024-12-22T12:14:34.354752Z","iopub.status.idle":"2024-12-22T12:14:37.182239Z","shell.execute_reply.started":"2024-12-22T12:14:34.354711Z","shell.execute_reply":"2024-12-22T12:14:37.181396Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Random Visualization of Some Images","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(0, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\nfor idx, ax in zip(random_idx, axes.ravel()):\n    img = Image.open(train_list[idx]).convert(\"RGB\")\n    label = 'Dog' if 'dog' in os.path.basename(train_list[idx]) else 'Cat'\n    ax.set_title(label)\n    ax.imshow(img)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:37.185310Z","iopub.execute_input":"2024-12-22T12:14:37.185645Z","iopub.status.idle":"2024-12-22T12:14:39.373901Z","shell.execute_reply.started":"2024-12-22T12:14:37.185619Z","shell.execute_reply":"2024-12-22T12:14:39.372612Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Further Split into Train and Validation","metadata":{}},{"cell_type":"code","source":"train_list, valid_list, train_labels, valid_labels = train_test_split(\n    train_list, labels_str, test_size=0.2, stratify=labels_str, random_state=seed\n)\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Validation Data: {len(valid_list)}\")\nprint(f\"Test Data: {len(test_split_list)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:39.375392Z","iopub.execute_input":"2024-12-22T12:14:39.376126Z","iopub.status.idle":"2024-12-22T12:14:39.403575Z","shell.execute_reply.started":"2024-12-22T12:14:39.376087Z","shell.execute_reply":"2024-12-22T12:14:39.402793Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Image Augmentation and Transformation\nPreprocessing steps for ViT-B/16, which typically expects 224x224 input size and ImageNet normalization.\n","metadata":{}},{"cell_type":"code","source":"\ntrain_transforms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomResizedCrop(224),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225]),\n])\n\nval_transforms = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225]),\n])\n\ntest_transforms = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225]),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:39.404683Z","iopub.execute_input":"2024-12-22T12:14:39.405228Z","iopub.status.idle":"2024-12-22T12:14:39.413173Z","shell.execute_reply.started":"2024-12-22T12:14:39.405192Z","shell.execute_reply":"2024-12-22T12:14:39.412413Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Custom Dataset Definition","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path).convert(\"RGB\")\n        img_transformed = self.transform(img)\n\n        label = 1 if 'dog' in os.path.basename(img_path) else 0\n\n        return img_transformed, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:39.414439Z","iopub.execute_input":"2024-12-22T12:14:39.414726Z","iopub.status.idle":"2024-12-22T12:14:39.427227Z","shell.execute_reply.started":"2024-12-22T12:14:39.414701Z","shell.execute_reply":"2024-12-22T12:14:39.426321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Creating the Datasets and DataLoaders","metadata":{}},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=val_transforms)\ntest_data = CatsDogsDataset(test_split_list, transform=test_transforms)\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True, num_workers=4)\nvalid_loader = DataLoader(dataset=valid_data, batch_size=batch_size, shuffle=False, num_workers=4)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=False, num_workers=4)\n\nprint(f\"Train Dataset size: {len(train_data)}\")\nprint(f\"Validation Dataset size: {len(valid_data)}\")\nprint(f\"Test Dataset size: {len(test_data)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:39.428389Z","iopub.execute_input":"2024-12-22T12:14:39.428707Z","iopub.status.idle":"2024-12-22T12:14:39.437326Z","shell.execute_reply.started":"2024-12-22T12:14:39.428679Z","shell.execute_reply":"2024-12-22T12:14:39.436483Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loading the Pre-trained ViT-B/16 Model from Torchvision","metadata":{}},{"cell_type":"code","source":"model = models.vit_b_16(weights=models.ViT_B_16_Weights.IMAGENET1K_V1)\nmodel.to(device)\n\n# Modify the final layer for binary classification\nnum_ftrs = model.heads.head.in_features\nmodel.heads.head = nn.Linear(num_ftrs, 2)\n\n# If multiple GPUs are available\nmodel = nn.DataParallel(model)\nmodel.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:39.438492Z","iopub.execute_input":"2024-12-22T12:14:39.439118Z","iopub.status.idle":"2024-12-22T12:14:43.208107Z","shell.execute_reply.started":"2024-12-22T12:14:39.439082Z","shell.execute_reply":"2024-12-22T12:14:43.207242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Training","metadata":{}},{"cell_type":"code","source":"\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=lr)\nscheduler = StepLR(optimizer, step_size=1, gamma=gamma)\n\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\nfor epoch in range(epochs):\n    epoch_loss = 0\n    epoch_accuracy = 0\n\n    model.train()\n    for data, label in tqdm(train_loader, desc=f\"Training Epoch {epoch+1}\"):\n        data = data.to(device)\n        label = label.to(device)\n\n        output = model(data)\n        loss = criterion(output, label)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        acc = (output.argmax(dim=1) == label).float().mean()\n        epoch_accuracy += acc.item() / len(train_loader)\n        epoch_loss += loss.item() / len(train_loader)\n\n    train_losses.append(epoch_loss)\n    train_accuracies.append(epoch_accuracy)\n\n    model.eval()\n    with torch.no_grad():\n        epoch_val_accuracy = 0\n        epoch_val_loss = 0\n        for data, label in tqdm(valid_loader, desc=f\"Validation Epoch {epoch+1}\"):\n            data = data.to(device)\n            label = label.to(device)\n\n            val_output = model(data)\n            val_loss = criterion(val_output, label)\n\n            acc = (val_output.argmax(dim=1) == label).float().mean()\n            epoch_val_accuracy += acc.item() / len(valid_loader)\n            epoch_val_loss += val_loss.item() / len(valid_loader)\n\n    val_losses.append(epoch_val_loss)\n    val_accuracies.append(epoch_val_accuracy)\n\n    print(\n        f\"Epoch: {epoch+1} - Loss: {epoch_loss:.4f} - Accuracy: {epoch_accuracy:.4f} - \"\n        f\"Val Loss: {epoch_val_loss:.4f} - Val Accuracy: {epoch_val_accuracy:.4f}\\n\"\n    )\n\n    scheduler.step()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:14:43.209200Z","iopub.execute_input":"2024-12-22T12:14:43.209515Z","iopub.status.idle":"2024-12-22T12:26:09.943140Z","shell.execute_reply.started":"2024-12-22T12:14:43.209490Z","shell.execute_reply":"2024-12-22T12:26:09.942098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot Training and Validation Accuracy and loss","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.title(\"Training and Validation Loss\")\nplt.plot(train_losses, label=\"Training\")\nplt.plot(val_losses, label=\"Validation\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()\n\nplt.figure(figsize=(10,5))\nplt.title(\"Training and Validation Accuracy\")\nplt.plot(train_accuracies, label=\"Training\")\nplt.plot(val_accuracies, label=\"Validation\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:26:09.944719Z","iopub.execute_input":"2024-12-22T12:26:09.945636Z","iopub.status.idle":"2024-12-22T12:26:10.466412Z","shell.execute_reply.started":"2024-12-22T12:26:09.945591Z","shell.execute_reply":"2024-12-22T12:26:10.465548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test Evaluation","metadata":{}},{"cell_type":"code","source":"with torch.no_grad():\n    test_accuracy = 0\n    test_loss = 0\n    for data, label in tqdm(test_loader, desc=\"Testing\"):\n        data = data.to(device)\n        label = label.to(device)\n\n        test_output = model(data)\n        loss = criterion(test_output, label)\n\n        acc = (test_output.argmax(dim=1) == label).float().mean()\n        test_accuracy += acc.item() / len(test_loader)\n        test_loss += loss.item() / len(test_loader)\n\n    print(f\"Test Loss: {test_loss:.4f} - Test Accuracy: {test_accuracy:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:26:10.467506Z","iopub.execute_input":"2024-12-22T12:26:10.467782Z","iopub.status.idle":"2024-12-22T12:26:44.236844Z","shell.execute_reply.started":"2024-12-22T12:26:10.467756Z","shell.execute_reply":"2024-12-22T12:26:44.235829Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Sample Predictions on Test Set","metadata":{}},{"cell_type":"code","source":"model.eval()\n\ndata_iter = iter(test_loader)\nimages, labels = next(data_iter)\n\nimages = images.to(device)\nlabels = labels.to(device)\n\noutputs = model(images)\n_, preds = torch.max(outputs, 1)\n\nimages = images.cpu()\nlabels = labels.cpu()\npreds = preds.cpu()\n\nfig, axes = plt.subplots(3, 3, figsize=(12, 12))\nfor i, ax in enumerate(axes.ravel()):\n    if i >= len(images):\n        break\n    # Denormalize the image for display\n    img = images[i].permute(1, 2, 0)  # Convert from CHW to HWC\n    img = img * torch.tensor([0.229, 0.224, 0.225]) + torch.tensor([0.485, 0.456, 0.406])\n    img = img.numpy()\n    img = np.clip(img, 0, 1)\n    ax.imshow(img)\n    gt = 'Dog' if labels[i] == 1 else 'Cat'\n    pred = 'Dog' if preds[i] == 1 else 'Cat'\n    ax.set_title(f\"Ground Truth: {gt} | Prediction: {pred}\")\n    ax.axis(\"off\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T12:26:44.238195Z","iopub.execute_input":"2024-12-22T12:26:44.238530Z","iopub.status.idle":"2024-12-22T12:26:47.155053Z","shell.execute_reply.started":"2024-12-22T12:26:44.238501Z","shell.execute_reply":"2024-12-22T12:26:47.154011Z"}},"outputs":[],"execution_count":null}]}