{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-14T09:27:18.939716Z","iopub.execute_input":"2024-01-14T09:27:18.940550Z","iopub.status.idle":"2024-01-14T09:27:18.945522Z","shell.execute_reply.started":"2024-01-14T09:27:18.940512Z","shell.execute_reply":"2024-01-14T09:27:18.944590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:27:18.947075Z","iopub.execute_input":"2024-01-14T09:27:18.947330Z","iopub.status.idle":"2024-01-14T09:27:18.959265Z","shell.execute_reply.started":"2024-01-14T09:27:18.947306Z","shell.execute_reply":"2024-01-14T09:27:18.958359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:27:18.961180Z","iopub.execute_input":"2024-01-14T09:27:18.961576Z","iopub.status.idle":"2024-01-14T09:27:19.959112Z","shell.execute_reply.started":"2024-01-14T09:27:18.961544Z","shell.execute_reply":"2024-01-14T09:27:19.958151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:27:19.960794Z","iopub.execute_input":"2024-01-14T09:27:19.961169Z","iopub.status.idle":"2024-01-14T09:27:19.968368Z","shell.execute_reply.started":"2024-01-14T09:27:19.961133Z","shell.execute_reply":"2024-01-14T09:27:19.967427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNISTDataset(Dataset):\n    def __init__(self, data_df:pd.DataFrame, transform=None, is_test=False):\n        super(MNISTDataset, self).__init__()\n        dataset = []\n            \n        for i, row in tqdm(data_df.iterrows(), total=len(data_df)):\n            data = row.to_numpy()\n            if is_test:\n                label = -1\n                image = data.reshape(28, 28).astype(np.uint8)\n            else:\n                label = data[0]\n                image = data[1:].reshape(28, 28).astype(np.uint8)\n            \n            if transform is not None:\n                image = transform(image)\n                    \n            dataset.append((image, label))\n        self.dataset = dataset\n        self.transform = transform\n        self.is_test = is_test\n    \n    def __len__(self):\n        return len(self.dataset)\n    \n    def __getitem__(self, i):\n        return self.dataset[i]","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:27:19.970534Z","iopub.execute_input":"2024-01-14T09:27:19.970838Z","iopub.status.idle":"2024-01-14T09:27:19.979551Z","shell.execute_reply.started":"2024-01-14T09:27:19.970807Z","shell.execute_reply":"2024-01-14T09:27:19.978630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntrain_data = data_train.drop('label', axis=1).values\ntrain_mean = train_data.mean()/255.\ntrain_std = train_data.std()/255.\neval_count = 1000\ntrain_transform = transform=transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomRotation(15),\n    transforms.RandomAffine(degrees=20, translate=(0.1,0.1), scale=(0.9, 1.1)),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[train_mean], std=[train_std]),\n])\nval_transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[train_mean], std=[train_std]),\n])\ntest_transform = val_transform\ntrain_dataset = MNISTDataset(data_train, train_transform) \neval_dataset = MNISTDataset(data_train.iloc[-eval_count:], val_transform)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:27:19.980552Z","iopub.execute_input":"2024-01-14T09:27:19.980826Z","iopub.status.idle":"2024-01-14T09:27:59.181361Z","shell.execute_reply.started":"2024-01-14T09:27:19.980803Z","shell.execute_reply":"2024-01-14T09:27:59.180431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntest_dataset = MNISTDataset(data_test, test_transform, is_test=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:27:59.182683Z","iopub.execute_input":"2024-01-14T09:27:59.183063Z","iopub.status.idle":"2024-01-14T09:28:10.248920Z","shell.execute_reply.started":"2024-01-14T09:27:59.183026Z","shell.execute_reply":"2024-01-14T09:28:10.248183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:10.250838Z","iopub.execute_input":"2024-01-14T09:28:10.251132Z","iopub.status.idle":"2024-01-14T09:28:10.279624Z","shell.execute_reply.started":"2024-01-14T09:28:10.251105Z","shell.execute_reply":"2024-01-14T09:28:10.278762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train.label.unique()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:10.280877Z","iopub.execute_input":"2024-01-14T09:28:10.281222Z","iopub.status.idle":"2024-01-14T09:28:10.288664Z","shell.execute_reply.started":"2024-01-14T09:28:10.281190Z","shell.execute_reply":"2024-01-14T09:28:10.287739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train[data_train.label == 1].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:10.292865Z","iopub.execute_input":"2024-01-14T09:28:10.293135Z","iopub.status.idle":"2024-01-14T09:28:10.330780Z","shell.execute_reply.started":"2024-01-14T09:28:10.293111Z","shell.execute_reply":"2024-01-14T09:28:10.329817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = data_train.iloc[1].to_numpy()\nlabel = row[0]\ndigit_img = row[1:].reshape(28, 28)\nprint(\"label:\",label)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:10.331864Z","iopub.execute_input":"2024-01-14T09:28:10.332169Z","iopub.status.idle":"2024-01-14T09:28:10.339939Z","shell.execute_reply.started":"2024-01-14T09:28:10.332144Z","shell.execute_reply":"2024-01-14T09:28:10.338963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(digit_img, interpolation='nearest', cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:10.341063Z","iopub.execute_input":"2024-01-14T09:28:10.341354Z","iopub.status.idle":"2024-01-14T09:28:10.573873Z","shell.execute_reply.started":"2024-01-14T09:28:10.341329Z","shell.execute_reply":"2024-01-14T09:28:10.573011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install einops","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:10.574917Z","iopub.execute_input":"2024-01-14T09:28:10.575179Z","iopub.status.idle":"2024-01-14T09:28:22.309739Z","shell.execute_reply.started":"2024-01-14T09:28:10.575156Z","shell.execute_reply":"2024-01-14T09:28:22.308653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from einops import rearrange\n\nclass Residual(nn.Module):\n    def __init__(self, fn):\n        super().__init__()\n        self.fn = fn\n\n    def forward(self, x, **kwargs):\n        return self.fn(x, **kwargs) + x\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.fn = fn\n\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Linear(hidden_dim, dim)\n        )\n\n    def forward(self, x):\n        return self.net(x)\n\nclass Attention(nn.Module):\n    def __init__(self, dim, heads=8):\n        super().__init__()\n        self.heads = heads\n        self.scale = dim ** -0.5\n\n        self.to_qkv = nn.Linear(dim, dim * 3, bias=False)\n        self.to_out = nn.Linear(dim, dim)\n\n    def forward(self, x, mask = None):\n        b, n, _, h = *x.shape, self.heads\n        qkv = self.to_qkv(x)\n        q, k, v = rearrange(qkv, 'b n (qkv h d) -> qkv b h n d', qkv=3, h=h)\n\n        dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale\n\n        if mask is not None:\n            mask = F.pad(mask.flatten(1), (1, 0), value = True)\n            assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions'\n            mask = mask[:, None, :] * mask[:, :, None]\n            dots.masked_fill_(~mask, float('-inf'))\n            del mask\n\n        attn = dots.softmax(dim=-1)\n\n        out = torch.einsum('bhij,bhjd->bhid', attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        out =  self.to_out(out)\n        return out\n\nclass Transformer(nn.Module):\n    def __init__(self, dim, depth, heads, mlp_dim):\n        super().__init__()\n        self.layers = nn.ModuleList([])\n        for _ in range(depth):\n            self.layers.append(nn.ModuleList([\n                Residual(PreNorm(dim, Attention(dim, heads = heads))),\n                Residual(PreNorm(dim, FeedForward(dim, mlp_dim)))\n            ]))\n\n    def forward(self, x, mask=None):\n        for attn, ff in self.layers:\n            x = attn(x, mask=mask)\n            x = ff(x)\n        return x\n    \nclass ViT(nn.Module):\n    def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, channels=3):\n        super().__init__()\n        assert image_size % patch_size == 0, 'image dimensions must be divisible by the patch size'\n        num_patches = (image_size // patch_size) ** 2\n        patch_dim = channels * patch_size ** 2\n\n        self.patch_size = patch_size\n\n        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n        self.patch_to_embedding = nn.Linear(patch_dim, dim)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n        self.transformer = Transformer(dim, depth, heads, mlp_dim)\n\n        self.to_cls_token = nn.Identity()\n\n        self.mlp_head = nn.Sequential(\n            nn.Linear(dim, mlp_dim),\n            nn.GELU(),\n            nn.Linear(mlp_dim, num_classes)\n        )\n\n    def forward(self, img, mask=None):\n        p = self.patch_size\n\n        x = rearrange(img, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1=p, p2=p)\n        x = self.patch_to_embedding(x)\n\n        cls_tokens = self.cls_token.expand(img.shape[0], -1, -1)\n        x = torch.cat((cls_tokens, x), dim=1)\n        x += self.pos_embedding\n        x = self.transformer(x, mask)\n\n        x = self.to_cls_token(x[:, 0])\n        return self.mlp_head(x)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:22.311546Z","iopub.execute_input":"2024-01-14T09:28:22.311902Z","iopub.status.idle":"2024-01-14T09:28:22.336627Z","shell.execute_reply.started":"2024-01-14T09:28:22.311865Z","shell.execute_reply":"2024-01-14T09:28:22.335755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(42)\n\nBATCH_SIZE_TRAIN = 100\nBATCH_SIZE_TEST = 1000\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE_TRAIN, shuffle=True)\neval_loader = torch.utils.data.DataLoader(eval_dataset, batch_size=BATCH_SIZE_TEST, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:22.337846Z","iopub.execute_input":"2024-01-14T09:28:22.338191Z","iopub.status.idle":"2024-01-14T09:28:22.357569Z","shell.execute_reply.started":"2024-01-14T09:28:22.338160Z","shell.execute_reply":"2024-01-14T09:28:22.356746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\n\ndef train_epoch(model, optimizer, data_loader, loss_history):\n    total_samples = len(data_loader.dataset)\n    model.train()\n\n    for i, (data, target) in enumerate(data_loader):\n        data = data.to(device)\n        target = target.to(device)\n        optimizer.zero_grad()\n        output = F.log_softmax(model(data), dim=1)\n        loss = F.nll_loss(output, target)\n        loss.backward()\n        optimizer.step()\n\n        if i % 100 == 0:\n            print('[' +  '{:5}'.format(i * len(data)) + '/' + '{:5}'.format(total_samples) +\n                  ' (' + '{:3.0f}'.format(100 * i / len(data_loader)) + '%)]  Loss: ' +\n                  '{:6.4f}'.format(loss.item()))\n            loss_history.append(loss.item())","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:22.359506Z","iopub.execute_input":"2024-01-14T09:28:22.359936Z","iopub.status.idle":"2024-01-14T09:28:22.367935Z","shell.execute_reply.started":"2024-01-14T09:28:22.359902Z","shell.execute_reply":"2024-01-14T09:28:22.367122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, data_loader, loss_history):\n    model.eval()\n    \n    total_samples = len(data_loader.dataset)\n    correct_samples = 0\n    total_loss = 0\n\n    with torch.no_grad():\n        for data, target in data_loader:\n            data = data.to(device)\n            target = target.to(device)\n            output = F.log_softmax(model(data), dim=1)\n            loss = F.nll_loss(output, target, reduction='sum')\n            _, pred = torch.max(output, dim=1)\n            \n            total_loss += loss.item()\n            correct_samples += pred.eq(target).sum()\n\n    avg_loss = total_loss / total_samples\n    loss_history.append(avg_loss)\n    print('\\nAverage test loss: ' + '{:.4f}'.format(avg_loss) +\n          '  Accuracy:' + '{:5}'.format(correct_samples) + '/' +\n          '{:5}'.format(total_samples) + ' (' +\n          '{:4.2f}'.format(100.0 * correct_samples / total_samples) + '%)\\n')","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:22.369088Z","iopub.execute_input":"2024-01-14T09:28:22.369350Z","iopub.status.idle":"2024-01-14T09:28:22.377837Z","shell.execute_reply.started":"2024-01-14T09:28:22.369326Z","shell.execute_reply":"2024-01-14T09:28:22.377065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\nmodel = ViT(image_size=28, patch_size=7, num_classes=10, channels=1,\n            dim=64, depth=6, heads=8, mlp_dim=128)\nmodel = model.to(device)\n# optimizer = torch.optim.Adam(model.parameters(), lr=0.003)\noptimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n\ntrain_loss_history, test_loss_history = [], []","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:22.378968Z","iopub.execute_input":"2024-01-14T09:28:22.379270Z","iopub.status.idle":"2024-01-14T09:28:22.400852Z","shell.execute_reply.started":"2024-01-14T09:28:22.379245Z","shell.execute_reply":"2024-01-14T09:28:22.399956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_EPOCHS = 100\n\n\nscheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.95)\n\nstart_time = time.time()\nfor epoch in range(1, N_EPOCHS + 1):\n    print('Epoch:', epoch,'LR:', scheduler.get_last_lr())\n    train_epoch(model, optimizer, train_loader, train_loss_history)\n    evaluate(model, eval_loader, test_loss_history)\n    scheduler.step()\n\nprint('Execution time:', '{:5.2f}'.format(time.time() - start_time), 'seconds')","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:28:22.401962Z","iopub.execute_input":"2024-01-14T09:28:22.402235Z","iopub.status.idle":"2024-01-14T09:36:35.726653Z","shell.execute_reply.started":"2024-01-14T09:28:22.402210Z","shell.execute_reply":"2024-01-14T09:36:35.725621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE_TEST, shuffle=False)\n\nsubmission = [['ImageId', 'Label']]\n\n# Turn off gradients for validation\nwith torch.no_grad():\n    model.eval()\n    image_id = 1\n    \n    for images, _ in submission_loader:\n        images = images.to(device)\n        output = F.log_softmax(model(images), dim=1)\n        _, pred = torch.max(output, dim=1)\n        \n        for prediction in pred:\n            #print([image_id, prediction.item()])\n            submission.append([image_id, prediction.item()])\n            image_id += 1","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:43:27.622504Z","iopub.execute_input":"2024-01-14T09:43:27.623199Z","iopub.status.idle":"2024-01-14T09:43:28.395521Z","shell.execute_reply.started":"2024-01-14T09:43:27.623163Z","shell.execute_reply":"2024-01-14T09:43:28.394690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(submission)\nsubmission_df.columns = submission_df.iloc[0]\nsubmission_df = submission_df.drop(0, axis=0)\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:43:29.217559Z","iopub.execute_input":"2024-01-14T09:43:29.218445Z","iopub.status.idle":"2024-01-14T09:43:29.240889Z","shell.execute_reply.started":"2024-01-14T09:43:29.218403Z","shell.execute_reply":"2024-01-14T09:43:29.239965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T09:43:33.266434Z","iopub.execute_input":"2024-01-14T09:43:33.266816Z","iopub.status.idle":"2024-01-14T09:43:33.330825Z","shell.execute_reply.started":"2024-01-14T09:43:33.266784Z","shell.execute_reply":"2024-01-14T09:43:33.329940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}