{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport pandas as pd\nfrom torch import nn\nfrom torch import optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport timeit\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-01T18:12:14.047914Z","iopub.execute_input":"2024-02-01T18:12:14.049233Z","iopub.status.idle":"2024-02-01T18:12:14.057618Z","shell.execute_reply.started":"2024-02-01T18:12:14.049174Z","shell.execute_reply":"2024-02-01T18:12:14.056374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_SEED = 42\nBATCH_SIZE = 512\nEPOCHS = 5\nLEARNING_RATE = 1e-4\nNUM_CLASSES = 10\nPATCH_SIZE = 4\nIMG_SIZE = 28\nIN_CHANNELS = 1\nNUM_HEADS = 8\nDROPOUT = 0.001\nHIDDEN_DIM = 768\nADAM_WEIGHT_DECAY = 0\nADAM_BETAS = (0.9, 0.999)\nACTIVATION=\"gelu\"\nNUM_ENCODERS = 4\nEMBED_DIM = (PATCH_SIZE ** 2) * IN_CHANNELS # 16\nNUM_PATCHES = (IMG_SIZE // PATCH_SIZE) ** 2 # 49\n\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\ntorch.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed_all(RANDOM_SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:14.059800Z","iopub.execute_input":"2024-02-01T18:12:14.061777Z","iopub.status.idle":"2024-02-01T18:12:14.077614Z","shell.execute_reply.started":"2024-02-01T18:12:14.061733Z","shell.execute_reply":"2024-02-01T18:12:14.076139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchEmbedding(nn.Module):\n    def __init__(self, embed_dim, patch_size, num_patches, dropout, in_channels):\n        super().__init__()\n        self.patcher = nn.Sequential(\n            nn.Conv2d(\n                in_channels=in_channels,\n                out_channels=embed_dim,\n                kernel_size=patch_size,\n                stride=patch_size,\n            ),                  \n            nn.Flatten(2))\n\n        self.cls_token = nn.Parameter(torch.randn(size=(1, in_channels, embed_dim)), requires_grad=True)\n        self.position_embeddings = nn.Parameter(torch.randn(size=(1, num_patches+1, embed_dim)), requires_grad=True)\n        self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, x):\n        cls_token = self.cls_token.expand(x.shape[0], -1, -1)\n\n        x = self.patcher(x).permute(0, 2, 1)\n        x = torch.cat([cls_token, x], dim=1)\n        x = self.position_embeddings + x \n        x = self.dropout(x)\n        return x\n    \nmodel = PatchEmbedding(EMBED_DIM, PATCH_SIZE, NUM_PATCHES, DROPOUT, IN_CHANNELS).to(device)\nx = torch.randn(512, 1, 28, 28).to(device)\nprint(model(x).shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:14.079593Z","iopub.execute_input":"2024-02-01T18:12:14.081504Z","iopub.status.idle":"2024-02-01T18:12:14.109708Z","shell.execute_reply.started":"2024-02-01T18:12:14.081452Z","shell.execute_reply":"2024-02-01T18:12:14.108234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViT(nn.Module):\n    def __init__(self, num_patches, img_size, num_classes, patch_size, embed_dim, num_encoders, num_heads, hidden_dim, dropout, activation, in_channels):\n        super().__init__()\n        self.embeddings_block = PatchEmbedding(embed_dim, patch_size, num_patches, dropout, in_channels)\n        \n        encoder_layer = nn.TransformerEncoderLayer(d_model=embed_dim, nhead=num_heads, dropout=dropout, activation=activation, batch_first=True, norm_first=True)\n        self.encoder_blocks = nn.TransformerEncoder(encoder_layer, num_layers=num_encoders)\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(normalized_shape=embed_dim),\n            nn.Linear(in_features=embed_dim, out_features=num_classes)\n        )\n\n    def forward(self, x):\n        x = self.embeddings_block(x)\n        x = self.encoder_blocks(x)\n        x = self.mlp_head(x[:, 0, :])  # Apply MLP on the CLS token only\n        return x\n\nmodel = ViT(NUM_PATCHES, IMG_SIZE, NUM_CLASSES, PATCH_SIZE, EMBED_DIM, NUM_ENCODERS, NUM_HEADS, HIDDEN_DIM, DROPOUT, ACTIVATION, IN_CHANNELS).to(device)\nx = torch.randn(512, 1, 28, 28).to(device)\nprint(model(x).shape) # BATCH_SIZE X NUM_CLASSES","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:14.113231Z","iopub.execute_input":"2024-02-01T18:12:14.114400Z","iopub.status.idle":"2024-02-01T18:12:17.385732Z","shell.execute_reply.started":"2024-02-01T18:12:14.114351Z","shell.execute_reply":"2024-02-01T18:12:17.384201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nsubmission_df = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:17.387073Z","iopub.execute_input":"2024-02-01T18:12:17.387434Z","iopub.status.idle":"2024-02-01T18:12:22.701243Z","shell.execute_reply.started":"2024-02-01T18:12:17.387383Z","shell.execute_reply":"2024-02-01T18:12:22.700237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:22.702478Z","iopub.execute_input":"2024-02-01T18:12:22.702787Z","iopub.status.idle":"2024-02-01T18:12:22.725633Z","shell.execute_reply.started":"2024-02-01T18:12:22.702761Z","shell.execute_reply":"2024-02-01T18:12:22.724482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:22.727560Z","iopub.execute_input":"2024-02-01T18:12:22.728225Z","iopub.status.idle":"2024-02-01T18:12:22.746718Z","shell.execute_reply.started":"2024-02-01T18:12:22.728193Z","shell.execute_reply":"2024-02-01T18:12:22.745728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:22.748170Z","iopub.execute_input":"2024-02-01T18:12:22.748547Z","iopub.status.idle":"2024-02-01T18:12:22.766251Z","shell.execute_reply.started":"2024-02-01T18:12:22.748510Z","shell.execute_reply":"2024-02-01T18:12:22.764914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=RANDOM_SEED, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:22.767676Z","iopub.execute_input":"2024-02-01T18:12:22.768057Z","iopub.status.idle":"2024-02-01T18:12:22.978511Z","shell.execute_reply.started":"2024-02-01T18:12:22.768026Z","shell.execute_reply":"2024-02-01T18:12:22.977345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNISTTrainDataset(Dataset):\n    def __init__(self, images, labels, indicies):\n        self.images = images\n        self.labels = labels\n        self.indicies = indicies\n        self.transform = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.RandomRotation(15),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.5])\n        ])\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx].reshape((28, 28)).astype(np.uint8)\n        label = self.labels[idx]\n        index = self.indicies[idx]\n        image = self.transform(image)\n        \n        return {\"image\": image, \"label\": label, \"index\": index}\n    \nclass MNISTValDataset(Dataset):\n    def __init__(self, images, labels, indicies):\n        self.images = images\n        self.labels = labels\n        self.indicies = indicies\n        self.transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.5])\n        ])\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx].reshape((28, 28)).astype(np.uint8)\n        label = self.labels[idx]\n        index = self.indicies[idx]\n        image = self.transform(image)\n        \n        return {\"image\": image, \"label\": label, \"index\": index}\n    \nclass MNISTSubmitDataset(Dataset):\n    def __init__(self, images, indicies):\n        self.images = images\n        self.indicies = indicies\n        self.transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.5])\n        ])\n    \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx].reshape((28, 28)).astype(np.uint8)\n        index = self.indicies[idx]\n        image = self.transform(image)\n        \n        return {\"image\": image, \"index\": index}","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:22.981948Z","iopub.execute_input":"2024-02-01T18:12:22.982319Z","iopub.status.idle":"2024-02-01T18:12:23.000771Z","shell.execute_reply.started":"2024-02-01T18:12:22.982286Z","shell.execute_reply":"2024-02-01T18:12:22.999373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nf, axarr = plt.subplots(1, 3)\n\ntrain_dataset = MNISTTrainDataset(train_df.iloc[:, 1:].values.astype(np.uint8), train_df.iloc[:, 0].values, train_df.index.values)\nprint(len(train_dataset))\nprint(train_dataset[0])\naxarr[0].imshow(train_dataset[0][\"image\"].squeeze(), cmap=\"gray\")\naxarr[0].set_title(\"Train Image\")\nprint(\"-\"*30)\n\nval_dataset = MNISTValDataset(val_df.iloc[:, 1:].values.astype(np.uint8), val_df.iloc[:, 0].values, val_df.index.values)\nprint(len(val_dataset))\nprint(val_dataset[0])\naxarr[1].imshow(val_dataset[0][\"image\"].squeeze(), cmap=\"gray\")\naxarr[1].set_title(\"Val Image\")\nprint(\"-\"*30)\n\ntest_dataset = MNISTSubmitDataset(test_df.values.astype(np.uint8), test_df.index.values)\nprint(len(test_dataset))\nprint(test_dataset[0])\naxarr[2].imshow(test_dataset[0][\"image\"].squeeze(), cmap=\"gray\")\naxarr[2].set_title(\"Test Image\")\nprint(\"-\"*30)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:23.002261Z","iopub.execute_input":"2024-02-01T18:12:23.002643Z","iopub.status.idle":"2024-02-01T18:12:23.556778Z","shell.execute_reply.started":"2024-02-01T18:12:23.002610Z","shell.execute_reply":"2024-02-01T18:12:23.555542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(dataset=train_dataset,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True)\n\nval_dataloader = DataLoader(dataset=val_dataset,\n                            batch_size=BATCH_SIZE,\n                            shuffle=True)\n\ntest_dataloader = DataLoader(dataset=test_dataset,\n                             batch_size=BATCH_SIZE,\n                             shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:23.558296Z","iopub.execute_input":"2024-02-01T18:12:23.558708Z","iopub.status.idle":"2024-02-01T18:12:23.568797Z","shell.execute_reply.started":"2024-02-01T18:12:23.558675Z","shell.execute_reply":"2024-02-01T18:12:23.567527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), betas=ADAM_BETAS, lr=LEARNING_RATE, weight_decay=ADAM_WEIGHT_DECAY)\n\nstart = timeit.default_timer()\nfor epoch in tqdm(range(EPOCHS), position=0, leave=True):\n    model.train()\n    train_labels = []\n    train_preds = []\n    train_running_loss = 0\n    for idx, img_label in enumerate(tqdm(train_dataloader, position=0, leave=True)):\n        img = img_label[\"image\"].float().to(device)\n        label = img_label[\"label\"].type(torch.uint8).to(device)\n        y_pred = model(img)\n        y_pred_label = torch.argmax(y_pred, dim=1)\n\n        train_labels.extend(label.cpu().detach())\n        train_preds.extend(y_pred_label.cpu().detach())\n        \n        loss = criterion(y_pred, label)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        train_running_loss += loss.item()\n    train_loss = train_running_loss / (idx + 1)\n\n    model.eval()\n    val_labels = []\n    val_preds = []\n    val_running_loss = 0\n    with torch.no_grad():\n        for idx, img_label in enumerate(tqdm(val_dataloader, position=0, leave=True)):\n            img = img_label[\"image\"].float().to(device)\n            label = img_label[\"label\"].type(torch.uint8).to(device)         \n            y_pred = model(img)\n            y_pred_label = torch.argmax(y_pred, dim=1)\n            \n            val_labels.extend(label.cpu().detach())\n            val_preds.extend(y_pred_label.cpu().detach())\n            \n            loss = criterion(y_pred, label)\n            val_running_loss += loss.item()\n    val_loss = val_running_loss / (idx + 1)\n\n    print(\"-\"*30)\n    print(f\"Train Loss EPOCH {epoch+1}: {train_loss:.4f}\")\n    print(f\"Valid Loss EPOCH {epoch+1}: {val_loss:.4f}\")\n    print(f\"Train Accuracy EPOCH {epoch+1}: {sum(1 for x,y in zip(train_preds, train_labels) if x == y) / len(train_labels):.4f}\")\n    print(f\"Valid Accuracy EPOCH {epoch+1}: {sum(1 for x,y in zip(val_preds, val_labels) if x == y) / len(val_labels):.4f}\")\n    print(\"-\"*30)\n\nstop = timeit.default_timer()\nprint(f\"Training Time: {stop-start:.2f}s\")","metadata":{"execution":{"iopub.status.busy":"2024-02-01T18:12:23.571120Z","iopub.execute_input":"2024-02-01T18:12:23.571571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nids = []\nimgs = []\nmodel.eval()\nwith torch.no_grad():\n    for idx, sample in enumerate(tqdm(test_dataloader, position=0, leave=True)):\n        img = sample[\"image\"].to(device)\n        ids.extend([int(i)+1 for i in sample[\"index\"]])\n        \n        outputs = model(img)\n        \n        imgs.extend(img.detach().cpu())\n        labels.extend([int(i) for i in torch.argmax(outputs, dim=1)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nf, axarr = plt.subplots(2, 3)\ncounter = 0\nfor i in range(2):\n    for j in range(3):\n        axarr[i][j].imshow(imgs[counter].squeeze(), cmap=\"gray\")\n        axarr[i][j].set_title(f\"Predicted {labels[counter]}\")\n        counter += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}