{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"},{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport pandas as pd\nfrom torch import nn\nfrom torch import optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport timeit\nfrom tqdm import tqdm\nimport os\nimport shutil\nfrom PIL import Image\nfrom tensorflow.keras.datasets import cifar10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T19:55:56.569001Z","iopub.execute_input":"2025-04-06T19:55:56.569285Z","iopub.status.idle":"2025-04-06T19:56:21.578735Z","shell.execute_reply.started":"2025-04-06T19:55:56.569255Z","shell.execute_reply":"2025-04-06T19:56:21.577838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RANDOM_SEED = 42\nBATCH_SIZE = 512\nEPOCHS = 200\nLEARNING_RATE = 1e-4\nNUM_CLASSES = 10\nPATCH_SIZE = 4\nIMG_SIZE = 32\nIN_CHANNELS = 3\nNUM_HEADS = 4\nDROPOUT = 0.1\nHIDDEN_DIM = 768\nADAM_WEIGHT_DECAY = 0 #previously 0\nADAM_BETAS = (0.9, 0.999)\nACTIVATION=\"gelu\"\nNUM_ENCODERS = 4 # increased from 4 to 8\nEMBED_DIM = (PATCH_SIZE ** 2) * IN_CHANNELS * 2 # Note that we *2 here to improve the performance of the model\nNUM_PATCHES = (IMG_SIZE // PATCH_SIZE) ** 2 # 49\n\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\ntorch.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed_all(RANDOM_SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T19:56:21.579905Z","iopub.execute_input":"2025-04-06T19:56:21.580461Z","iopub.status.idle":"2025-04-06T19:56:21.672706Z","shell.execute_reply.started":"2025-04-06T19:56:21.580434Z","shell.execute_reply":"2025-04-06T19:56:21.672076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PatchEmbedding(nn.Module):\n    def __init__(self, embed_dim, patch_size, num_patches, dropout, in_channels):\n        super().__init__()\n        self.patcher = nn.Sequential(\n            nn.Conv2d(\n                in_channels=in_channels,\n                out_channels=embed_dim,\n                kernel_size=patch_size,\n                stride=patch_size,\n            ),                  \n            nn.Flatten(2))\n\n        self.cls_token = nn.Parameter(torch.randn(size=(1, 1, embed_dim)), requires_grad=True)\n        self.position_embeddings = nn.Parameter(torch.randn(size=(1, num_patches+1, embed_dim)), requires_grad=True)\n        self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, x):\n        cls_token = self.cls_token.expand(x.shape[0], -1, -1)\n\n        x = self.patcher(x).permute(0, 2, 1)\n        x = torch.cat([cls_token, x], dim=1)\n        x = self.position_embeddings + x \n        x = self.dropout(x)\n        return x\n    \nmodel = PatchEmbedding(EMBED_DIM, PATCH_SIZE, NUM_PATCHES, DROPOUT, IN_CHANNELS).to(device)\nx = torch.randn(512, 3, 32, 32).to(device)\nprint(model(x).shape)","metadata":{"execution":{"iopub.status.busy":"2025-04-06T19:56:21.673615Z","iopub.execute_input":"2025-04-06T19:56:21.673834Z","iopub.status.idle":"2025-04-06T19:56:22.875783Z","shell.execute_reply.started":"2025-04-06T19:56:21.673816Z","shell.execute_reply":"2025-04-06T19:56:22.875022Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ViT(nn.Module):\n    def __init__(self, num_patches, img_size, num_classes, patch_size, embed_dim, num_encoders, num_heads, hidden_dim, dropout, activation, in_channels):\n        super().__init__()\n        self.embeddings_block = PatchEmbedding(embed_dim, patch_size, num_patches, dropout, in_channels)\n        \n        encoder_layer = nn.TransformerEncoderLayer(d_model=embed_dim, nhead=num_heads, dropout=dropout, activation=activation, batch_first=True, norm_first=True)\n        self.encoder_blocks = nn.TransformerEncoder(encoder_layer, num_layers=num_encoders)\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(normalized_shape=embed_dim),\n            nn.Linear(in_features=embed_dim, out_features=num_classes)\n        )\n\n    def forward(self, x):\n        x = self.embeddings_block(x)\n        x = self.encoder_blocks(x)\n        x = self.mlp_head(x[:, 0, :])  # Apply MLP on the CLS token only\n        return x\n\nmodel = ViT(NUM_PATCHES, IMG_SIZE, NUM_CLASSES, PATCH_SIZE, EMBED_DIM, NUM_ENCODERS, NUM_HEADS, HIDDEN_DIM, DROPOUT, ACTIVATION, IN_CHANNELS).to(device)\nx = torch.randn(512, 3, 32, 32).to(device)\nprint(model(x).shape) # BATCH_SIZE X NUM_CLASSES","metadata":{"execution":{"iopub.status.busy":"2025-04-06T19:56:22.877754Z","iopub.execute_input":"2025-04-06T19:56:22.877974Z","iopub.status.idle":"2025-04-06T19:56:23.201486Z","shell.execute_reply.started":"2025-04-06T19:56:22.877956Z","shell.execute_reply":"2025-04-06T19:56:23.200722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load CIFAR-10 from Keras\n(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n\n# Flatten labels because Keras gives shape (50000, 1)\ny_train = y_train.flatten()\ny_test = y_test.flatten()\n\n# Split train into train/val\ntrain_x, val_x, train_y, val_y = train_test_split(\n    x_train, y_train, test_size=0.1, stratify=y_train, random_state=42\n)\n\ntrain_indices = np.arange(len(train_x))\nval_indices = np.arange(len(val_x))\ntest_indices = np.arange(len(x_test))","metadata":{"execution":{"iopub.status.busy":"2025-04-06T19:56:23.202841Z","iopub.execute_input":"2025-04-06T19:56:23.203106Z","iopub.status.idle":"2025-04-06T19:56:28.02458Z","shell.execute_reply.started":"2025-04-06T19:56:23.203086Z","shell.execute_reply":"2025-04-06T19:56:28.023836Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision.transforms import RandAugment\n\n# Define Dataset Class\nclass CIFAR10ArrayDataset(Dataset):\n    def __init__(self, images, labels, indices, train=True):\n        self.images = images\n        self.labels = labels\n        self.indices = indices\n        \n        if train:\n            self.transform = transforms.Compose([\n                transforms.RandomCrop(32, padding=4),  # ← Add Random Crop first\n                transforms.RandomHorizontalFlip(),\n                RandAugment(),  # ← Add RandAugment here!\n                transforms.ToTensor(),\n                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n            ])\n        else:\n            self.transform = transforms.Compose([\n                transforms.ToTensor(),\n                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n            ])\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        img = Image.fromarray(self.images[idx])  # (H, W, 3) array → PIL Image\n        img = self.transform(img)\n        label = self.labels[idx]\n        index = self.indices[idx]\n        \n        return {\"image\": img, \"label\": label, \"index\": index}","metadata":{"execution":{"iopub.status.busy":"2025-04-06T19:56:28.025381Z","iopub.execute_input":"2025-04-06T19:56:28.025623Z","iopub.status.idle":"2025-04-06T19:56:28.032208Z","shell.execute_reply.started":"2025-04-06T19:56:28.025605Z","shell.execute_reply":"2025-04-06T19:56:28.031337Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create Datasets\ntrain_dataset = CIFAR10ArrayDataset(train_x, train_y, train_indices, train=True)\nval_dataset = CIFAR10ArrayDataset(val_x, val_y, val_indices, train=False)\ntest_dataset = CIFAR10ArrayDataset(x_test, y_test, test_indices, train=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T19:56:28.03307Z","iopub.execute_input":"2025-04-06T19:56:28.033456Z","iopub.status.idle":"2025-04-06T19:56:28.068166Z","shell.execute_reply.started":"2025-04-06T19:56:28.03342Z","shell.execute_reply":"2025-04-06T19:56:28.067275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# some of examples\n\nimport random\n\nrandom_indices = random.sample(range(50000), 100)  \n\nplt.figure(figsize=(10, 10))\n\nfor i, idx in enumerate(random_indices):\n    plt.subplot(10, 10, i + 1)\n    plt.imshow(x_train[idx])\n    plt.axis(\"off\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T19:56:28.069114Z","iopub.execute_input":"2025-04-06T19:56:28.069406Z","iopub.status.idle":"2025-04-06T19:56:30.794439Z","shell.execute_reply.started":"2025-04-06T19:56:28.069381Z","shell.execute_reply":"2025-04-06T19:56:30.79329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pick some images from the datasets\ntrain_sample = train_dataset[0]\nval_sample = val_dataset[0]\ntest_sample = test_dataset[0]\n\n# Plot\nplt.figure(figsize=(10,3))\nf, axarr = plt.subplots(1, 3)\n\n# Train\naxarr[0].imshow(train_sample[\"image\"].permute(1, 2, 0) * 0.5 + 0.5)  # unnormalize for visualization\naxarr[0].set_title(f\"Train Image - Label: {train_sample['label']}\")\naxarr[0].axis('off')\n\n# Val\naxarr[1].imshow(val_sample[\"image\"].permute(1, 2, 0) * 0.5 + 0.5)  # unnormalize\naxarr[1].set_title(f\"Val Image - Label: {val_sample['label']}\")\naxarr[1].axis('off')\n\n# Test\naxarr[2].imshow(test_sample[\"image\"].permute(1, 2, 0) * 0.5 + 0.5)  # unnormalize\naxarr[2].set_title(f\"Test Image - Label: {test_sample['label']}\")\naxarr[2].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T19:56:30.795286Z","iopub.execute_input":"2025-04-06T19:56:30.795544Z","iopub.status.idle":"2025-04-06T19:56:31.044991Z","shell.execute_reply.started":"2025-04-06T19:56:30.795524Z","shell.execute_reply":"2025-04-06T19:56:31.044216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataloader = DataLoader(dataset=train_dataset,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True, pin_memory = True)\n\nval_dataloader = DataLoader(dataset=val_dataset,\n                            batch_size=BATCH_SIZE,\n                            shuffle=True, pin_memory = True)\n\n# test_dataloader = DataLoader(dataset=test_dataset,\n#                              batch_size=BATCH_SIZE,\n#                              shuffle=False, pin_memory = True)","metadata":{"execution":{"iopub.status.busy":"2025-04-06T19:56:31.045866Z","iopub.execute_input":"2025-04-06T19:56:31.046212Z","iopub.status.idle":"2025-04-06T19:56:31.050637Z","shell.execute_reply.started":"2025-04-06T19:56:31.046178Z","shell.execute_reply":"2025-04-06T19:56:31.049815Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), betas=ADAM_BETAS, lr=LEARNING_RATE, weight_decay=ADAM_WEIGHT_DECAY)\n\nstart = timeit.default_timer()\nfor epoch in tqdm(range(EPOCHS), position=0, leave=True):\n    model.train()\n    train_labels = []\n    train_preds = []\n    train_running_loss = 0\n    for idx, img_label in enumerate(tqdm(train_dataloader, position=0, leave=True)):\n        img = img_label[\"image\"].float().to(device)\n        label = img_label[\"label\"].type(torch.uint8).to(device)\n        y_pred = model(img)\n        y_pred_label = torch.argmax(y_pred, dim=1)\n\n        train_labels.extend(label.cpu().detach())\n        train_preds.extend(y_pred_label.cpu().detach())\n        \n        loss = criterion(y_pred, label)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        train_running_loss += loss.item()\n    train_loss = train_running_loss / (idx + 1)\n\n    model.eval()\n    val_labels = []\n    val_preds = []\n    val_running_loss = 0\n    with torch.no_grad():\n        for idx, img_label in enumerate(tqdm(val_dataloader, position=0, leave=True)):\n            img = img_label[\"image\"].float().to(device)\n            label = img_label[\"label\"].type(torch.uint8).to(device)         \n            y_pred = model(img)\n            y_pred_label = torch.argmax(y_pred, dim=1)\n            \n            val_labels.extend(label.cpu().detach())\n            val_preds.extend(y_pred_label.cpu().detach())\n            \n            loss = criterion(y_pred, label)\n            val_running_loss += loss.item()\n    val_loss = val_running_loss / (idx + 1)\n\n    print(\"-\"*30)\n    print(f\"Train Loss EPOCH {epoch+1}: {train_loss:.4f}\")\n    print(f\"Valid Loss EPOCH {epoch+1}: {val_loss:.4f}\")\n    print(f\"Train Accuracy EPOCH {epoch+1}: {sum(1 for x,y in zip(train_preds, train_labels) if x == y) / len(train_labels):.4f}\")\n    print(f\"Valid Accuracy EPOCH {epoch+1}: {sum(1 for x,y in zip(val_preds, val_labels) if x == y) / len(val_labels):.4f}\")\n    print(\"-\"*30)\n\nstop = timeit.default_timer()\nprint(f\"Training Time: {stop-start:.2f}s\")","metadata":{"execution":{"iopub.status.busy":"2025-04-06T19:56:31.051651Z","iopub.execute_input":"2025-04-06T19:56:31.052025Z","iopub.status.idle":"2025-04-06T23:16:49.731524Z","shell.execute_reply.started":"2025-04-06T19:56:31.05199Z","shell.execute_reply":"2025-04-06T23:16:49.730728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T23:16:49.732275Z","iopub.execute_input":"2025-04-06T23:16:49.73252Z","iopub.status.idle":"2025-04-06T23:16:49.760051Z","shell.execute_reply.started":"2025-04-06T23:16:49.732499Z","shell.execute_reply":"2025-04-06T23:16:49.759233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = []\nids = []\nimgs = []\nmodel.eval()\nwith torch.no_grad():\n    for idx, sample in enumerate(tqdm(test_dataloader, position=0, leave=True)):\n        img = sample[\"image\"].to(device)\n        ids.extend([int(i)+1 for i in sample[\"index\"]])\n        \n        outputs = model(img)\n        \n        imgs.extend(img.detach().cpu())\n        labels.extend([int(i) for i in torch.argmax(outputs, dim=1)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T23:16:49.762134Z","iopub.execute_input":"2025-04-06T23:16:49.762373Z","iopub.status.idle":"2025-04-06T23:16:50.023991Z","shell.execute_reply.started":"2025-04-06T23:16:49.762353Z","shell.execute_reply":"2025-04-06T23:16:50.022832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure()\nf, axarr = plt.subplots(2, 3)\ncounter = 0\nfor i in range(2):\n    for j in range(3):\n        axarr[i][j].imshow(imgs[counter].squeeze(), cmap=\"gray\")\n        axarr[i][j].set_title(f\"Predicted {labels[counter]}\")\n        counter += 1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.DataFrame(list(zip(ids, labels)),\n               columns =[\"ImageId\", \"Label\"])\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}