{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport torchmetrics\nimport pytorch_lightning as pl\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split as tts\n\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor\nfrom pytorch_lightning.loggers import CSVLogger\nfrom torchmetrics import Accuracy\nfrom pytorch_lightning.callbacks import Callback\nimport os\nimport copy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:26.235453Z","iopub.execute_input":"2026-09-13T10:36:26.236108Z","iopub.status.idle":"2026-09-13T10:36:34.416649Z","shell.execute_reply.started":"2026-09-13T10:36:26.236074Z","shell.execute_reply":"2026-09-13T10:36:34.415684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train = pd.read_csv('/kaggle/input/competitions/digit-recognizer/train.csv')\nfeature_train_data    = data_train.iloc[:,1:]\nlabel_train_data      = data_train.iloc[:,0]\n\n\ndata_test = pd.read_csv('/kaggle/input/competitions/digit-recognizer/test.csv')\nX_test    = data_test.iloc[:,:]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:34.418252Z","iopub.execute_input":"2026-09-13T10:36:34.418920Z","iopub.status.idle":"2026-09-13T10:36:37.627436Z","shell.execute_reply.started":"2026-09-13T10:36:34.418878Z","shell.execute_reply":"2026-09-13T10:36:37.626829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MNIST_Preprocessor(Dataset):\n    def __init__(self,X,Y=None,transform=None,has_label=True):\n        super().__init__()\n        self.feature   = X\n        self.has_label = has_label\n        if self.has_label:\n            self.label     = Y\n            \n        self.transform = transform\n\n    def __len__(self):\n        return len(self.feature)\n\n    def __getitem__(self,idx):\n        image = self.feature.iloc[idx,:].values.astype(np.uint8).reshape(28,28,1)\n        if self.has_label:\n            label = self.label.iloc[idx]\n\n        if self.transform:\n            image = self.transform(image)\n\n        if self.has_label:\n            return image,label\n\n        \n        return image\n\n\ntransform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomRotation(15),\n    transforms.ToTensor()\n])\n        \n        \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:37.628287Z","iopub.execute_input":"2026-09-13T10:36:37.628498Z","iopub.status.idle":"2026-09-13T10:36:37.635310Z","shell.execute_reply.started":"2026-09-13T10:36:37.628476Z","shell.execute_reply":"2026-09-13T10:36:37.634495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train,X_val,Y_train,Y_val = tts(feature_train_data,label_train_data,test_size=0.2,stratify=label_train_data,random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:37.636745Z","iopub.execute_input":"2026-09-13T10:36:37.637188Z","iopub.status.idle":"2026-09-13T10:36:37.872203Z","shell.execute_reply.started":"2026-09-13T10:36:37.637150Z","shell.execute_reply":"2026-09-13T10:36:37.871570Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = MNIST_Preprocessor(X=X_train,Y=Y_train,transform=transform)\nval_dataset   = MNIST_Preprocessor(X=X_val,Y=Y_val,transform=transform)\ntest_dataset  = MNIST_Preprocessor(X=X_test,transform=transform,has_label=False)\n\n\ntrain_loader  = DataLoader(train_dataset,batch_size=64,shuffle=True,num_workers=3)\nval_loader    = DataLoader(val_dataset,batch_size=64,shuffle=False,num_workers=3)\ntest_loader   = DataLoader(test_dataset,batch_size=64,shuffle=False,num_workers=3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:37.873998Z","iopub.execute_input":"2026-09-13T10:36:37.874249Z","iopub.status.idle":"2026-09-13T10:36:37.880500Z","shell.execute_reply.started":"2026-09-13T10:36:37.874221Z","shell.execute_reply":"2026-09-13T10:36:37.879832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PatchEmbedding(torch.nn.Module):\n    def __init__(self,img_size=28,patch_size=7,input_channel = 1, embedding_dim=32):\n        super().__init__()\n\n        self.img_size   = img_size\n        self.patch_size = patch_size\n        self.embed_size = embedding_dim\n\n        self.patch_no   = (img_size//patch_size)**2\n\n        self.projection = nn.Conv2d(input_channel,self.embed_size,kernel_size=self.patch_size,stride=self.patch_size)\n\n    def forward(self,x):\n        B,C,H,W = x.shape\n\n        x = self.projection(x)\n        x = x.flatten(2)\n        x = x.transpose(1,2)\n        return x\n\n\nclass MHA(nn.Module):\n    def __init__(self,num_heads=4,embedding_dim=32):\n        super().__init__()\n        self.num_heads = num_heads\n        self.embed_dim = embedding_dim\n        self.head_dim  = (self.embed_dim//self.num_heads)\n\n        self.scale     = self.head_dim**(-0.5)\n\n        self.qkv       = nn.Linear(self.embed_dim,self.embed_dim*3,bias=True)\n\n        self.proj      = nn.Linear(self.embed_dim,self.embed_dim)\n\n    def forward(self,x):\n        Batch,Num_tokens,Embedding_dim = x.shape\n\n        qkv                            = self.qkv(x)\n        qkv                            = qkv.reshape(Batch,Num_tokens,3,self.num_heads,self.head_dim)\n\n        qkv                            = qkv.permute(2,0,3,1,4) ## (3,Batch,num_heads,Num_tokens,head_dim)\n\n        q,k,v                          = qkv[0],qkv[1],qkv[2]    ## (Batch,num_heads,Num_tokens,head_dim)\n\n        k_t                            = k.transpose(-2,-1)      ## (Batch,num_heads,head_dim,Num_tokens)\n\n        query_key_cosine_similarity    = torch.matmul(q,k_t)*self.scale  ## (Batch,num_heads,Num_tokens,Num_tokens)\n\n        attention_scores               = query_key_cosine_similarity.softmax(dim=-1)\n\n        output_value                   = torch.matmul(attention_scores,v)  ## (Batch,num_heads,Num_tokens,head_dim)\n\n        combined_heads                 = output_value.transpose(1,2).reshape(Batch,Num_tokens,Embedding_dim)\n\n        projected_output               = self.proj(combined_heads)\n\n        return projected_output\n\n         \n\nclass MLP(nn.Module):\n    def __init__(self,embedding_dim=32,mlp_ratio = 2):\n        super().__init__()\n        self.embedding_dim = embedding_dim\n        self.fc1 = nn.Linear(self.embedding_dim,self.embedding_dim*mlp_ratio)\n        self.gelu= nn.GELU()\n        self.fc2 = nn.Linear(self.embedding_dim*mlp_ratio,self.embedding_dim)\n\n    def forward(self,x):\n        x = self.fc1(x)\n        x = self.gelu(x)\n        x = self.fc2(x)\n\n        return x\n\n\nclass TransformerEncoder(nn.Module):\n    def __init__(self,embedding_dim=32,mlp_ratio=2):\n        super().__init__()\n        self.norm1      = nn.LayerNorm(embedding_dim)\n        self.attention  = MHA(embedding_dim=embedding_dim)\n        self.norm2      = nn.LayerNorm(embedding_dim)\n        self.mlp        = MLP(embedding_dim=embedding_dim)\n\n    def forward(self,x):\n        x = x+self.attention(self.norm1(x))\n        x = x+self.mlp(self.norm2(x))\n        return x\n        \n\nclass VisionTransformers(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.patch_embedding = PatchEmbedding()\n        self.cls_token       = nn.Parameter(torch.zeros(1,1,32))\n        self.pos_encoding    = nn.Parameter(torch.zeros(1,17,32))\n        self.blocks = nn.ModuleList([\n            TransformerEncoder()\n            for _ in range(3)\n        ])\n\n        self.norm = nn.LayerNorm(32)\n        self.head = nn.Linear(32, 10)\n\n    def forward(self,x):\n      B          = x.shape[0]\n      x          = self.patch_embedding(x)\n      cls_tokens = self.cls_token.expand(B, -1, -1)\n      x          = torch.cat((cls_tokens, x), dim=1)\n      x          = x+self.pos_encoding\n      for block in self.blocks:\n            x = block(x)\n\n      x = self.norm(x)\n      return self.head(x[:, 0])\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:37.881506Z","iopub.execute_input":"2026-09-13T10:36:37.881922Z","iopub.status.idle":"2026-09-13T10:36:37.900873Z","shell.execute_reply.started":"2026-09-13T10:36:37.881876Z","shell.execute_reply":"2026-09-13T10:36:37.900048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Using device:\", device) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:37.901985Z","iopub.execute_input":"2026-09-13T10:36:37.902280Z","iopub.status.idle":"2026-09-13T10:36:37.950808Z","shell.execute_reply.started":"2026-09-13T10:36:37.902240Z","shell.execute_reply":"2026-09-13T10:36:37.950001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------------------------------------------\n# DEVICE\n# ------------------------------------------------------------------\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Using device:\", device)\nif torch.cuda.is_available():\n    print(\"GPU:\", torch.cuda.get_device_name(0))\n    print(\"GPU count:\", torch.cuda.device_count())\n\n# ------------------------------------------------------------------\n# MODEL / LOSS / METRICS\n# ------------------------------------------------------------------\nmodel = VisionTransformers().to(device)\n\ncriterion = nn.CrossEntropyLoss()\n\ntrain_acc_metric = Accuracy(task=\"multiclass\", num_classes=10).to(device)\nval_acc_metric = Accuracy(task=\"multiclass\", num_classes=10).to(device)\n\noptimizer = optim.Adam(model.parameters(), lr=1e-3)\n\nEPOCHS = 60\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n\n# ------------------------------------------------------------------\n# CHECKPOINT / EARLY STOPPING STATE\n# ------------------------------------------------------------------\nos.makedirs(\"checkpoints\", exist_ok=True)\n\nbest_val_acc = -1.0\nbest_model_state = None\nbest_epoch = -1\npatience = 20\nepochs_no_improve = 0\n\n# ------------------------------------------------------------------\n# TRAIN LOOP\n# ------------------------------------------------------------------\nfor epoch in range(EPOCHS):\n\n    # ---------------- TRAIN ----------------\n    model.train()\n    train_acc_metric.reset()\n    running_train_loss = 0.0\n    n_train_batches = 0\n\n    for x, y in train_loader:\n        x, y = x.to(device), y.to(device)\n\n        optimizer.zero_grad()\n        logits = model(x)\n        loss = criterion(logits, y)\n        loss.backward()\n        optimizer.step()\n\n        preds = logits.argmax(dim=1)\n        train_acc_metric(preds, y)\n\n        running_train_loss += loss.item()\n        n_train_batches += 1\n\n    scheduler.step()\n\n    train_loss = running_train_loss / n_train_batches\n    train_acc = train_acc_metric.compute().item()\n\n    # ---------------- VALIDATE ----------------\n    model.eval()\n    val_acc_metric.reset()\n    running_val_loss = 0.0\n    n_val_batches = 0\n\n    with torch.no_grad():\n        for x, y in val_loader:\n            x, y = x.to(device), y.to(device)\n\n            logits = model(x)\n            loss = criterion(logits, y)\n\n            preds = logits.argmax(dim=1)\n            val_acc_metric(preds, y)\n\n            running_val_loss += loss.item()\n            n_val_batches += 1\n\n    val_loss = running_val_loss / n_val_batches\n    val_acc = val_acc_metric.compute().item()\n\n    current_lr = optimizer.param_groups[0][\"lr\"]\n\n    print(\n        f\"Epoch {epoch+1:02d}/{EPOCHS} | \"\n        f\"train_loss={train_loss:.4f} train_acc={train_acc:.4f} | \"\n        f\"val_loss={val_loss:.4f} val_acc={val_acc:.4f} | \"\n        f\"lr={current_lr:.6f}\"\n    )\n\n    # ---------------- CHECKPOINT (save_top_k=1, monitor val_acc, mode max) ----------------\n    if val_acc > best_val_acc:\n        best_val_acc = val_acc\n        best_epoch = epoch\n        best_model_state = copy.deepcopy(model.state_dict())\n        epochs_no_improve = 0\n\n        ckpt_path = f\"checkpoints/best_vit_mnist-epoch{epoch:02d}-val_acc{val_acc:.4f}.pt\"\n        torch.save(\n            {\n                \"epoch\": epoch,\n                \"model_state_dict\": best_model_state,\n                \"optimizer_state_dict\": optimizer.state_dict(),\n                \"val_acc\": val_acc,\n            },\n            ckpt_path,\n        )\n        best_model_path = ckpt_path\n    else:\n        epochs_no_improve += 1\n\n    # ---------------- EARLY STOPPING (patience=10) ----------------\n    if epochs_no_improve >= patience:\n        print(f\"Early stopping triggered at epoch {epoch+1} \"\n              f\"(no improvement for {patience} epochs).\")\n        break\n\n# ------------------------------------------------------------------\n# BEST MODEL INFORMATION\n# ------------------------------------------------------------------\nprint(f\"Best val_acc: {best_val_acc:.4f}\")\nprint(f\"Best checkpoint path: {best_model_path}\")\n\n# ------------------------------------------------------------------\n# LOAD BEST CHECKPOINT\n# ------------------------------------------------------------------\nbest_model = VisionTransformers().to(device)\ncheckpoint = torch.load(best_model_path, map_location=device)\nbest_model.load_state_dict(checkpoint[\"model_state_dict\"])\nbest_model.eval()\n\n# ------------------------------------------------------------------\n# PREDICT ON LABEL-LESS TEST SET\n# ------------------------------------------------------------------\nall_preds = []\n\nwith torch.no_grad():\n    for batch in test_loader:\n        x = batch[0] if isinstance(batch, (list, tuple)) else batch\n        x = x.to(device)\n\n        logits = best_model(x)\n        preds = logits.argmax(dim=1)\n        all_preds.append(preds.cpu())\n\npredictions = torch.cat(all_preds)\n\nprint(\"Prediction shape:\", predictions.shape)\nprint(\"First 20 predictions:\")\nprint(predictions[:20])\n\n# ------------------------------------------------------------------\n# KAGGLE SUBMISSION CSV (Digit Recognizer format: ImageId, Label)\n# ------------------------------------------------------------------\nsample_submission_path = \"/kaggle/input/digit-recognizer/sample_submission.csv\"\n\nif os.path.exists(sample_submission_path):\n    sample_submission = pd.read_csv(sample_submission_path)\n    submission = sample_submission.copy()\n    submission[\"Label\"] = predictions.numpy()\nelse:\n    # Fallback if sample_submission.csv isn't available/mounted:\n    # ImageId is 1-indexed, matching Digit Recognizer's convention.\n    submission = pd.DataFrame({\n        \"ImageId\": range(1, len(predictions) + 1),\n        \"Label\": predictions.numpy()\n    })\n\nsubmission_path = \"submission.csv\"\nsubmission.to_csv(submission_path, index=False)\n\nprint(f\"Submission saved to: {submission_path}\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-13T10:36:37.951670Z","iopub.execute_input":"2026-09-13T10:36:37.951901Z","iopub.status.idle":"2026-09-13T10:41:19.869934Z","shell.execute_reply.started":"2026-09-13T10:36:37.951878Z","shell.execute_reply":"2026-09-13T10:41:19.868970Z"}},"outputs":[],"execution_count":null}]}