{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11677871,"sourceType":"datasetVersion","datasetId":7329424},{"sourceId":11678295,"sourceType":"datasetVersion","datasetId":7329718}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CNNs vs ViTs: Comparing AlexNet and Google's 2020 Vision Transformer\n\n### This notebook explores the performance differences between two influential architectures in computer vision:\n\n#### - **AlexNet (2012)** – A pioneering convolutional neural network architecture that helped establish deep learning in computer vision.\n#### - **Vision Transformer (ViT/16) (2020)** – A transformer-based model that introduced self-attention mechanisms to image patches, redefining modern image recognition.","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport matplotlib.pyplot as plt\nimport pickle\nimport numpy as np\nimport sys\nimport os\nimport time\nimport pandas as pd\nfrom torch.utils.data import DataLoader, TensorDataset\n\nsys.path.append(\"/kaggle/input/vit-pytorch/vit-pytorch\")\nfrom vit_pytorch.vit_for_small_dataset import ViT","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T02:39:25.735878Z","iopub.execute_input":"2025-05-05T02:39:25.736170Z","iopub.status.idle":"2025-05-05T02:39:25.741027Z","shell.execute_reply.started":"2025-05-05T02:39:25.736149Z","shell.execute_reply":"2025-05-05T02:39:25.740369Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Load Cifar_10 Dataset","metadata":{}},{"cell_type":"code","source":"def unpickle(file):\n    with open(file, 'rb') as fo:\n        return pickle.load(fo, encoding='bytes')\n\ndef load_cifar10():\n    base_path = '/kaggle/input/cifar-10'\n    X_train = []\n    y_train = []\n\n    for i in range(1, 6):\n        batch = unpickle(os.path.join(base_path, f'data_batch_{i}'))\n        X = batch[b'data'].reshape(-1, 3, 32, 32).astype(np.float32) / 255.0\n        y = np.array(batch[b'labels'])\n        X_train.append(X)\n        y_train.append(y)\n\n    X_train = np.concatenate(X_train, axis=0)\n    y_train = np.concatenate(y_train, axis=0)\n\n    test_batch = unpickle(os.path.join(base_path, 'test_batch'))\n    X_test = test_batch[b'data'].reshape(-1, 3, 32, 32).astype(np.float32) / 255.0\n    y_test = np.array(test_batch[b'labels'])\n\n    return X_train, y_train, X_test, y_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T02:39:25.742384Z","iopub.execute_input":"2025-05-05T02:39:25.742614Z","iopub.status.idle":"2025-05-05T02:39:25.758428Z","shell.execute_reply.started":"2025-05-05T02:39:25.742597Z","shell.execute_reply":"2025-05-05T02:39:25.757841Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Convolutional Neural Network","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN,self).__init__()\n        self.features = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3,padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2,stride=2),\n\n            nn.Conv2d(64, 128, kernel_size=3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n\n            nn.Conv2d(128, 256, kernel_size=3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n        )\n\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(256 * 4 * 4, 512),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, 10)\n        )\n\n    def forward(self,x):\n        x = self.features(x)\n        x = self.classifier(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T02:39:25.759060Z","iopub.execute_input":"2025-05-05T02:39:25.759248Z","iopub.status.idle":"2025-05-05T02:39:25.774663Z","shell.execute_reply.started":"2025-05-05T02:39:25.759232Z","shell.execute_reply":"2025-05-05T02:39:25.773985Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Visual Transformer","metadata":{}},{"cell_type":"code","source":"#vit_model = ViT(\n#    image_size=32,\n#    patch_size=4,\n#    num_classes=10,\n#    dim=128,\n#   depth=6,\n#   heads=8,\n#   mlp_dim=256,\n#   channels=3,\n#   dropout=0.1,\n#   emb_dropout=0.1\n#)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T02:39:25.775321Z","iopub.execute_input":"2025-05-05T02:39:25.775567Z","iopub.status.idle":"2025-05-05T02:39:25.790269Z","shell.execute_reply.started":"2025-05-05T02:39:25.775544Z","shell.execute_reply":"2025-05-05T02:39:25.789709Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Testing Phase","metadata":{}},{"cell_type":"code","source":"X_train, y_train, X_test, y_test = load_cifar10()\n\nX_train = torch.tensor(X_train, dtype=torch.float32)\ny_train = torch.tensor(y_train, dtype=torch.long)\n\nX_test = torch.tensor(X_test, dtype=torch.float32)\ny_test = torch.tensor(y_test, dtype=torch.long)\n\ntrain_loader = DataLoader(TensorDataset(X_train, y_train), batch_size=64, shuffle=True)\ntest_loader = DataLoader(TensorDataset(X_test, y_test), batch_size=64)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ncriterion = nn.CrossEntropyLoss()\nprint(\"Device:\",device)\n\ndef train_model(model, optimizer, train_loader, test_loader, epochs=5):\n    model.to(device)\n    train_acc_history = []\n    test_acc_history = []\n\n    overall_start = time.time()\n\n    for epoch in range(epochs):\n        start_time = time.time()\n\n        model.train()\n        total_loss = 0\n        correct = 0\n\n        for x, y in train_loader:\n            x, y = x.to(device), y.to(device)\n\n            optimizer.zero_grad()\n            out = model(x)\n            loss = criterion(out, y)\n            loss.backward()\n            optimizer.step()\n\n            total_loss += loss.item()\n            correct += (out.argmax(1) == y).sum().item()\n\n        train_acc = correct / len(train_loader.dataset)\n        test_acc = test_model(model, test_loader, verbose=False)\n\n        train_acc_history.append(train_acc)\n        test_acc_history.append(test_acc)\n\n        elapsed = time.time() - start_time\n        total_elapsed = time.time() - overall_start\n\n        print(f\"Epoch {epoch+1}: Loss={total_loss:.4f}, Train Acc={train_acc*100:.2f}%, Test Acc={test_acc*100:.2f}%, Time: {elapsed:.2f}s, Total Elapsed: {total_elapsed:.2f}s\")\n\n    return train_acc_history, test_acc_history\n\ndef test_model(model, test_loader, verbose=True):\n    model.eval()\n    correct = 0\n    with torch.no_grad():\n        for x, y in test_loader:\n            x, y = x.to(device), y.to(device)\n            out = model(x)\n            correct += (out.argmax(1) == y).sum().item()\n    acc = correct / len(test_loader.dataset)\n    if verbose:\n        print(f\"Test Accuracy: {acc*100:.2f}%\")\n    return acc\n\ncnn_model = CNN()\nvit_model = ViT(\n    image_size=32,\n    patch_size=4,\n    num_classes=10,\n    dim=128,\n    depth=6,\n    heads=8,\n    mlp_dim=256,\n    channels=3,\n    dropout=0.1,\n    emb_dropout=0.1\n)\n\noptimizer_cnn = optim.Adam(cnn_model.parameters(), lr=1e-3)\noptimizer_vit = optim.Adam(vit_model.parameters(), lr=1e-3)\n\nprint(\"Training CNN:\")\ncnn_accs = train_model(cnn_model, optimizer_cnn, train_loader, test_loader)\n\nprint(\"\\nTraining ViT:\")\nvit_accs = train_model(vit_model, optimizer_vit, train_loader, test_loader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T02:39:25.791769Z","iopub.execute_input":"2025-05-05T02:39:25.791934Z","iopub.status.idle":"2025-05-05T02:41:48.761691Z","shell.execute_reply.started":"2025-05-05T02:39:25.791922Z","shell.execute_reply":"2025-05-05T02:41:48.761052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, cnn_test_acc = cnn_accs\n_, vit_test_acc = vit_accs\n\n# Plot\nplt.figure(figsize=(8, 5))\nplt.plot(cnn_test_acc, label='CNN Test Accuracy', marker='o')\nplt.plot(vit_test_acc, label='ViT Test Accuracy', marker='s')\nplt.title(\"CNN vs ViT Test Accuracy Over Epochs\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.ylim(0, 1)\nplt.grid(True)\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-05T02:41:48.762425Z","iopub.execute_input":"2025-05-05T02:41:48.762604Z","iopub.status.idle":"2025-05-05T02:41:49.013840Z","shell.execute_reply.started":"2025-05-05T02:41:48.762590Z","shell.execute_reply":"2025-05-05T02:41:49.013101Z"}},"outputs":[],"execution_count":null}]}