{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-14T12:33:27.160316Z","iopub.execute_input":"2024-08-14T12:33:27.160908Z","iopub.status.idle":"2024-08-14T12:33:27.165263Z","shell.execute_reply.started":"2024-08-14T12:33:27.160875Z","shell.execute_reply":"2024-08-14T12:33:27.164302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset preparation:\n\n- Read train dataset\n- Create validation dataset\n- Create Pytorch custom dataset preparation class\n- Create Pytorch dataloaders\n\n","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:27.167175Z","iopub.execute_input":"2024-08-14T12:33:27.167609Z","iopub.status.idle":"2024-08-14T12:33:30.065472Z","shell.execute_reply.started":"2024-08-14T12:33:27.167576Z","shell.execute_reply":"2024-08-14T12:33:30.064478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.066758Z","iopub.execute_input":"2024-08-14T12:33:30.067113Z","iopub.status.idle":"2024-08-14T12:33:30.084950Z","shell.execute_reply.started":"2024-08-14T12:33:30.067082Z","shell.execute_reply":"2024-08-14T12:33:30.083896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.098302Z","iopub.execute_input":"2024-08-14T12:33:30.098699Z","iopub.status.idle":"2024-08-14T12:33:30.109307Z","shell.execute_reply.started":"2024-08-14T12:33:30.098664Z","shell.execute_reply":"2024-08-14T12:33:30.108266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df , val_df = train_test_split(train,test_size=0.2 , random_state=0,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.110542Z","iopub.execute_input":"2024-08-14T12:33:30.111137Z","iopub.status.idle":"2024-08-14T12:33:30.331457Z","shell.execute_reply.started":"2024-08-14T12:33:30.111105Z","shell.execute_reply":"2024-08-14T12:33:30.330660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.332697Z","iopub.execute_input":"2024-08-14T12:33:30.333054Z","iopub.status.idle":"2024-08-14T12:33:30.337602Z","shell.execute_reply.started":"2024-08-14T12:33:30.333022Z","shell.execute_reply":"2024-08-14T12:33:30.336684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self , data , dataset_type):\n        self.images = data.iloc[:,1:].values\n        self.labels = data.iloc[: , 0].values\n        self.dataset_type = dataset_type\n        self.train_transform = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.RandomRotation(15),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.5])])\n        self.valid_transform = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.5])])\n    \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self,idx):\n        image = self.images[idx].reshape(28,28).astype(np.uint8)\n        label = self.labels[idx]\n        if self.dataset_type=='train':\n            image = self.train_transform(image)\n        else:\n            image = self.valid_transform(image)\n        \n        return image,label\n            ","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.338989Z","iopub.execute_input":"2024-08-14T12:33:30.339548Z","iopub.status.idle":"2024-08-14T12:33:30.350489Z","shell.execute_reply.started":"2024-08-14T12:33:30.339522Z","shell.execute_reply":"2024-08-14T12:33:30.349664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = CustomDataset(train_df, 'train')\nvalid_ds = CustomDataset(val_df, 'valid')","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.354883Z","iopub.execute_input":"2024-08-14T12:33:30.355262Z","iopub.status.idle":"2024-08-14T12:33:30.362715Z","shell.execute_reply.started":"2024-08-14T12:33:30.355226Z","shell.execute_reply":"2024-08-14T12:33:30.361913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_ds) , len(valid_ds)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.363695Z","iopub.execute_input":"2024-08-14T12:33:30.363937Z","iopub.status.idle":"2024-08-14T12:33:30.374553Z","shell.execute_reply.started":"2024-08-14T12:33:30.363916Z","shell.execute_reply":"2024-08-14T12:33:30.373679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(train_ds.labels)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.375787Z","iopub.execute_input":"2024-08-14T12:33:30.376073Z","iopub.status.idle":"2024-08-14T12:33:30.385882Z","shell.execute_reply.started":"2024-08-14T12:33:30.376046Z","shell.execute_reply":"2024-08-14T12:33:30.385024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 512\ntrain_dataloader = DataLoader(dataset=train_ds,\n                              batch_size=BATCH_SIZE,\n                              shuffle=True)\n\nval_dataloader = DataLoader(dataset=valid_ds,\n                            batch_size=BATCH_SIZE,\n                            shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.386951Z","iopub.execute_input":"2024-08-14T12:33:30.387244Z","iopub.status.idle":"2024-08-14T12:33:30.395726Z","shell.execute_reply.started":"2024-08-14T12:33:30.387220Z","shell.execute_reply":"2024-08-14T12:33:30.394860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image,label in train_dataloader:\n    print(image.shape)\n    print(label.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.396836Z","iopub.execute_input":"2024-08-14T12:33:30.397088Z","iopub.status.idle":"2024-08-14T12:33:30.576394Z","shell.execute_reply.started":"2024-08-14T12:33:30.397055Z","shell.execute_reply":"2024-08-14T12:33:30.575467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Vision Transformer","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\n","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:33:30.577612Z","iopub.execute_input":"2024-08-14T12:33:30.577946Z","iopub.status.idle":"2024-08-14T12:33:30.582244Z","shell.execute_reply.started":"2024-08-14T12:33:30.577920Z","shell.execute_reply":"2024-08-14T12:33:30.581274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Parameters","metadata":{}},{"cell_type":"code","source":"NUM_CLASSES = 10\nPATCH_SIZE = 4\nIMG_SIZE = 28\nIN_CHANNELS = 1\nNUM_HEADS = 8\nDROPOUT = 0.001\nHIDDEN_DIM = 768\nACTIVATION=\"gelu\"\nNUM_ENCODERS = 8\nEMBED_DIM = (PATCH_SIZE ** 2) * IN_CHANNELS # 16\nNUM_PATCHES = (IMG_SIZE // PATCH_SIZE) ** 2 # 49\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:26.649037Z","iopub.execute_input":"2024-08-14T12:41:26.649409Z","iopub.status.idle":"2024-08-14T12:41:26.658784Z","shell.execute_reply.started":"2024-08-14T12:41:26.649380Z","shell.execute_reply":"2024-08-14T12:41:26.657857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### PatchEmbedding class","metadata":{}},{"cell_type":"code","source":"class PatchEmbedding(nn.Module):\n    def __init__(self, embed_dim, patch_size, num_patches, dropout, in_channels):\n        super().__init__()\n        self.patcher = nn.Sequential(\n            nn.Conv2d(\n                in_channels=in_channels,\n                out_channels=embed_dim,\n                kernel_size=patch_size,\n                stride=patch_size,\n            ),                  \n            nn.Flatten(2))\n\n        self.cls_token = nn.Parameter(torch.randn(size=(1, 1, embed_dim)), requires_grad=True)\n        self.position_embeddings = nn.Parameter(torch.randn(size=(1, num_patches+1, embed_dim)), requires_grad=True)\n        self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, x):\n        cls_token = self.cls_token.expand(x.shape[0], -1, -1)\n\n        x = self.patcher(x).permute(0, 2, 1)\n        x = torch.cat([cls_token, x], dim=1)\n        x = self.position_embeddings + x \n        x = self.dropout(x)\n        return x\n    \nmodel = PatchEmbedding(EMBED_DIM, PATCH_SIZE, NUM_PATCHES, DROPOUT, IN_CHANNELS).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:28.503911Z","iopub.execute_input":"2024-08-14T12:41:28.504276Z","iopub.status.idle":"2024-08-14T12:41:28.515010Z","shell.execute_reply.started":"2024-08-14T12:41:28.504248Z","shell.execute_reply":"2024-08-14T12:41:28.514031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:28.998946Z","iopub.execute_input":"2024-08-14T12:41:28.999305Z","iopub.status.idle":"2024-08-14T12:41:29.006534Z","shell.execute_reply.started":"2024-08-14T12:41:28.999277Z","shell.execute_reply":"2024-08-14T12:41:29.005684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\npatched = model(image.to(device))","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:29.473925Z","iopub.execute_input":"2024-08-14T12:41:29.474283Z","iopub.status.idle":"2024-08-14T12:41:29.479834Z","shell.execute_reply.started":"2024-08-14T12:41:29.474255Z","shell.execute_reply":"2024-08-14T12:41:29.478870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patched.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:29.973073Z","iopub.execute_input":"2024-08-14T12:41:29.973997Z","iopub.status.idle":"2024-08-14T12:41:29.979562Z","shell.execute_reply.started":"2024-08-14T12:41:29.973964Z","shell.execute_reply":"2024-08-14T12:41:29.978683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Vision Transformer","metadata":{}},{"cell_type":"code","source":"class VissionTransformer(nn.Module):\n    def __init__(self, num_patches, img_size, num_classes, patch_size, embed_dim, num_encoders, \n                 num_heads, hidden_dim, dropout, activation, in_channels):\n        super().__init__()\n        self.embeddings_block = PatchEmbedding(embed_dim, patch_size, num_patches, dropout, in_channels)\n        \n        encoder_layer = nn.TransformerEncoderLayer(d_model=embed_dim, nhead=num_heads, dropout=dropout, \n                                                   activation=activation, batch_first=True, norm_first=True)\n        self.encoder_blocks = nn.TransformerEncoder(encoder_layer, num_layers=num_encoders)\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(normalized_shape=embed_dim),\n            nn.Linear(in_features=embed_dim, out_features=num_classes)\n        )\n\n    def forward(self, x):\n        x = self.embeddings_block(x)    # Patch embedding blocks\n        x = self.encoder_blocks(x)     # Encoder blocker with Multi-head attention\n        x = self.mlp_head(x[:, 0, :])  # Apply classification on the CLS token only - MLP layer\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:30.988978Z","iopub.execute_input":"2024-08-14T12:41:30.989731Z","iopub.status.idle":"2024-08-14T12:41:30.997468Z","shell.execute_reply.started":"2024-08-14T12:41:30.989700Z","shell.execute_reply":"2024-08-14T12:41:30.996538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = VissionTransformer(NUM_PATCHES, IMG_SIZE, NUM_CLASSES, PATCH_SIZE, EMBED_DIM,\n                           NUM_ENCODERS, NUM_HEADS, HIDDEN_DIM,\n                           DROPOUT, ACTIVATION, IN_CHANNELS).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:31.559073Z","iopub.execute_input":"2024-08-14T12:41:31.559461Z","iopub.status.idle":"2024-08-14T12:41:31.584197Z","shell.execute_reply.started":"2024-08-14T12:41:31.559429Z","shell.execute_reply":"2024-08-14T12:41:31.583254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nmpl_output = model(image.to(device))","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:32.204095Z","iopub.execute_input":"2024-08-14T12:41:32.204470Z","iopub.status.idle":"2024-08-14T12:41:32.221900Z","shell.execute_reply.started":"2024-08-14T12:41:32.204439Z","shell.execute_reply":"2024-08-14T12:41:32.221118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mpl_output.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:32.879244Z","iopub.execute_input":"2024-08-14T12:41:32.879638Z","iopub.status.idle":"2024-08-14T12:41:32.885773Z","shell.execute_reply.started":"2024-08-14T12:41:32.879589Z","shell.execute_reply":"2024-08-14T12:41:32.884750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mpl_output[0]","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:33.873814Z","iopub.execute_input":"2024-08-14T12:41:33.874619Z","iopub.status.idle":"2024-08-14T12:41:33.882585Z","shell.execute_reply.started":"2024-08-14T12:41:33.874584Z","shell.execute_reply":"2024-08-14T12:41:33.881690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loss function and Optimizer","metadata":{}},{"cell_type":"code","source":"# loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:35.239217Z","iopub.execute_input":"2024-08-14T12:41:35.239956Z","iopub.status.idle":"2024-08-14T12:41:35.245471Z","shell.execute_reply.started":"2024-08-14T12:41:35.239920Z","shell.execute_reply":"2024-08-14T12:41:35.244391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Training","metadata":{}},{"cell_type":"code","source":"EPOCHS = 100 ## run the model for 100 epochs\n\nfor epoch in tqdm(range(EPOCHS)): \n    train_loss, valid_loss = [], []\n    ## training part \n    model.train()\n    for images, labels in train_dataloader:\n        \n        #moving data into the device\n        images , labels = images.to(device) , labels.to(device)\n        \n        optimizer.zero_grad()\n        ## 1. forward propagation\n        outputs = model(images)\n        \n        ## 2. loss calculation\n        loss = criterion(outputs, labels)\n        \n        ## 3. backward propagation\n        loss.backward()\n        \n        ## 4. weight optimization\n        optimizer.step()\n        \n        train_loss.append(loss.item())\n        \n    ## evaluation part\n    with torch.no_grad():\n        model.eval()\n        for images, labels in val_dataloader:\n            images , labels = images.to(device) , labels.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            valid_loss.append(loss.item())\n    print (\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:41:36.594020Z","iopub.execute_input":"2024-08-14T12:41:36.594373Z","iopub.status.idle":"2024-08-14T12:50:09.964574Z","shell.execute_reply.started":"2024-08-14T12:41:36.594345Z","shell.execute_reply":"2024-08-14T12:50:09.963651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:50:09.966466Z","iopub.execute_input":"2024-08-14T12:50:09.966904Z","iopub.status.idle":"2024-08-14T12:50:10.109981Z","shell.execute_reply.started":"2024-08-14T12:50:09.966870Z","shell.execute_reply":"2024-08-14T12:50:10.109093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference using trained model","metadata":{}},{"cell_type":"code","source":"predictions = []\nmodel.eval()\n\nwith torch.no_grad():\n    for images, labels in tqdm(val_dataloader):\n        images = images.to(device)\n                \n        outputs = model(images)\n        predictions.extend([int(i) for i in torch.argmax(outputs, dim=1)])","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:50:18.897206Z","iopub.execute_input":"2024-08-14T12:50:18.897832Z","iopub.status.idle":"2024-08-14T12:50:21.372700Z","shell.execute_reply.started":"2024-08-14T12:50:18.897800Z","shell.execute_reply":"2024-08-14T12:50:21.371591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(val_df['label'].values,predictions)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T12:50:22.766671Z","iopub.execute_input":"2024-08-14T12:50:22.767365Z","iopub.status.idle":"2024-08-14T12:50:22.777892Z","shell.execute_reply.started":"2024-08-14T12:50:22.767324Z","shell.execute_reply":"2024-08-14T12:50:22.776816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}