{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30528,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-11T08:18:03.460439Z","iopub.execute_input":"2024-01-11T08:18:03.460843Z","iopub.status.idle":"2024-01-11T08:18:03.477825Z","shell.execute_reply.started":"2024-01-11T08:18:03.460812Z","shell.execute_reply":"2024-01-11T08:18:03.476683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\nimport random\nimport timeit\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:03.479546Z","iopub.execute_input":"2024-01-11T08:18:03.479836Z","iopub.status.idle":"2024-01-11T08:18:07.182673Z","shell.execute_reply.started":"2024-01-11T08:18:03.479807Z","shell.execute_reply":"2024-01-11T08:18:07.181637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_SEED = 1234\nBATCH_SIZE = 512\nEPOCHS = 100\nLEARNING_RATE = 0.001\nNUM_CLASSES = 10\nPATCH_SIZE = 4\nIMG_SIZE = 28\nIN_CHANNELS = 1\nNUM_HEADS = 8\nDROPOUT = 0.001\nHIDDEN_DIM = 768\nADAM_WEIGHT_DECAY = 0\nADAM_BETAS = (0.9, 0.999)\nACTIVATION=\"gelu\"\nNUM_ENCODERS = 4\nEMBED_DIM = (PATCH_SIZE ** 2) * IN_CHANNELS # 16\nNUM_PATCH = (IMG_SIZE // PATCH_SIZE) ** 2 # 49\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\ntorch.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed_all(RANDOM_SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:07.183997Z","iopub.execute_input":"2024-01-11T08:18:07.184612Z","iopub.status.idle":"2024-01-11T08:18:07.228085Z","shell.execute_reply.started":"2024-01-11T08:18:07.184564Z","shell.execute_reply":"2024-01-11T08:18:07.227129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchEmbedding(nn.Module):\n    def __init__(self, embed_dim, patch_size, num_patch, dropout, in_channels):\n        super().__init__()\n        self.patcher = nn.Sequential(\n            nn.Conv2d(in_channels=in_channels, out_channels=embed_dim, kernel_size=patch_size, stride=patch_size),\n            nn.Flatten(2)\n        )\n        self.cls_token = nn.Parameter(torch.randn(size=(1, in_channels, embed_dim)), requires_grad=True)\n        self.position_embedding = nn.Parameter(torch.randn(size=(1, num_patch+1, embed_dim)), requires_grad=True)\n        self.dropout = nn.Dropout(p=dropout)\n    \n    def forward(self, x):\n        cls_token = self.cls_token.expand(x.shape[0], -1, -1)\n        x = self.patcher(x).permute(0, 2, 1)\n        x = torch.cat([cls_token, x], dim=1)\n        x = self.position_embedding + x\n        x = self.dropout(x)\n        \n        return x\n\nmodel = PatchEmbedding(EMBED_DIM, PATCH_SIZE, NUM_PATCH, DROPOUT, IN_CHANNELS)\nmodel.to(device)\nx = torch.randn(512, 1, 28, 28).to(device)\nprint(model(x).shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:07.230703Z","iopub.execute_input":"2024-01-11T08:18:07.231355Z","iopub.status.idle":"2024-01-11T08:18:07.959745Z","shell.execute_reply.started":"2024-01-11T08:18:07.231327Z","shell.execute_reply":"2024-01-11T08:18:07.958814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViT(nn.Module):\n    def __init__(self, num_patch, img_size, num_classes, patch_size, embed_dim, num_heads, num_encoders, hidden_dim, dropout, activation, in_channels):\n        super().__init__()\n        self.embedding_block = PatchEmbedding(embed_dim, patch_size, num_patch, dropout, in_channels)\n        encoder_layer = nn.TransformerEncoderLayer(d_model=embed_dim, nhead=num_heads, dropout=dropout, activation=activation, batch_first=True, norm_first=True)\n        self.encoder_blocks = nn.TransformerEncoder(encoder_layer, num_layers=num_encoders)\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(normalized_shape=embed_dim),\n            nn.Linear(in_features=embed_dim, out_features=num_classes)\n        )\n    \n    def forward(self, x):\n        x = self.embedding_block(x)\n        x = self.encoder_blocks(x)\n        x = self.mlp_head(x[:, 0, :])\n        return x\n\nmodel = ViT(NUM_PATCH, IMG_SIZE, NUM_CLASSES, PATCH_SIZE, EMBED_DIM, NUM_HEADS, NUM_ENCODERS, HIDDEN_DIM, DROPOUT, ACTIVATION, IN_CHANNELS)\nmodel.to(device)\nx = torch.randn(512, 1, 28, 28).to(device)\nprint(model(x).shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:07.960971Z","iopub.execute_input":"2024-01-11T08:18:07.961259Z","iopub.status.idle":"2024-01-11T08:18:08.103900Z","shell.execute_reply.started":"2024-01-11T08:18:07.961232Z","shell.execute_reply":"2024-01-11T08:18:08.102797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:08.105431Z","iopub.execute_input":"2024-01-11T08:18:08.106108Z","iopub.status.idle":"2024-01-11T08:18:13.710117Z","shell.execute_reply.started":"2024-01-11T08:18:08.106072Z","shell.execute_reply":"2024-01-11T08:18:13.709301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:13.711242Z","iopub.execute_input":"2024-01-11T08:18:13.711524Z","iopub.status.idle":"2024-01-11T08:18:13.734543Z","shell.execute_reply.started":"2024-01-11T08:18:13.711488Z","shell.execute_reply":"2024-01-11T08:18:13.733689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:13.735714Z","iopub.execute_input":"2024-01-11T08:18:13.736110Z","iopub.status.idle":"2024-01-11T08:18:13.759564Z","shell.execute_reply.started":"2024-01-11T08:18:13.736078Z","shell.execute_reply":"2024-01-11T08:18:13.758592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=RANDOM_SEED, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:13.760650Z","iopub.execute_input":"2024-01-11T08:18:13.760986Z","iopub.status.idle":"2024-01-11T08:18:13.956816Z","shell.execute_reply.started":"2024-01-11T08:18:13.760951Z","shell.execute_reply":"2024-01-11T08:18:13.956012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Train_Dataset(Dataset):\n    def __init__(self, images, labels, indices):\n        self.images = images\n        self.labels = labels\n        self.indices = indices\n        self.transform = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.RandomRotation(15),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5],[0.5])\n        ])\n    \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx].reshape((28, 28)).astype(np.uint8)\n        label = self.labels[idx]\n        index = self.indices[idx]\n        image = self.transform(image)\n        \n        return {\"image\": image, \"label\": label, \"index\": index}\n\nclass Test_Dataset(Dataset):\n    def __init__(self, images, indicies):\n        self.images = images\n        self.indicies = indicies\n        self.transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.5])\n        ])\n    \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx].reshape((28, 28)).astype(np.uint8)\n        index = self.indicies[idx]\n        image = self.transform(image)\n        \n        return {\"image\": image, \"index\": index}","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:13.960647Z","iopub.execute_input":"2024-01-11T08:18:13.960932Z","iopub.status.idle":"2024-01-11T08:18:13.971097Z","shell.execute_reply.started":"2024-01-11T08:18:13.960908Z","shell.execute_reply":"2024-01-11T08:18:13.970033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nf, axes = plt.subplots(1, 3)\n\ntrain_ds = Train_Dataset(train_df.iloc[:, 1:].values.astype(np.uint8), train_df.iloc[:, 0].values, train_df.index.values)\nprint(len(train_ds))\naxes[0].imshow(train_ds[0][\"image\"].squeeze(), cmap=\"gray\")\naxes[0].set_title(\"Train Image\")\n\nval_ds = Train_Dataset(val_df.iloc[:, 1:].values.astype(np.uint8), val_df.iloc[:, 0].values, val_df.index.values)\nprint(len(val_ds))\naxes[1].imshow(val_ds[0][\"image\"].squeeze(), cmap=\"gray\")\naxes[1].set_title(\"Validation Image\")\n\ntest_ds = Test_Dataset(test_df.iloc[:, 0:].values.astype(np.uint8), test_df.index.values)\nprint(len(test_ds))\naxes[2].imshow(test_ds[0][\"image\"].squeeze(), cmap=\"gray\")\naxes[2].set_title(\"Test Image\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:13.972309Z","iopub.execute_input":"2024-01-11T08:18:13.972577Z","iopub.status.idle":"2024-01-11T08:18:14.452549Z","shell.execute_reply.started":"2024-01-11T08:18:13.972553Z","shell.execute_reply":"2024-01-11T08:18:14.451621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(dataset=train_ds, batch_size=BATCH_SIZE, shuffle=True)\nval_dataloader = DataLoader(dataset=val_ds, batch_size=BATCH_SIZE, shuffle=True)\ntest_dataloader = DataLoader(dataset=test_ds, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:14.453883Z","iopub.execute_input":"2024-01-11T08:18:14.454260Z","iopub.status.idle":"2024-01-11T08:18:14.460724Z","shell.execute_reply.started":"2024-01-11T08:18:14.454231Z","shell.execute_reply":"2024-01-11T08:18:14.459806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_func = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), betas=ADAM_BETAS, lr=LEARNING_RATE, weight_decay=ADAM_WEIGHT_DECAY)\n\nfor epoch in tqdm(range(EPOCHS), position=0, leave=True):\n    model.train()\n    train_labels = []\n    train_preds = []\n    cost = 0\n    for idx, data in enumerate(tqdm(train_dataloader, position=0, leave=True)):\n        img = data[\"image\"].float().to(device)\n        label = data[\"label\"].type(torch.uint8).to(device)\n        y_pred = model(img)\n        y_pred_label = torch.argmax(y_pred, dim=1)\n        train_labels.extend(label.cpu().detach())\n        train_preds.extend(y_pred_label.cpu().detach())\n        \n        loss = loss_func(y_pred, label)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        cost += loss.item()\n    train_cost = cost / (1+idx)\n    \n    model.eval()\n    val_labels = []\n    val_preds = []\n    val_cost = 0\n    with torch.no_grad():\n        for idx, data in enumerate(tqdm(val_dataloader, position=0, leave=True)):\n            img = data[\"image\"].float().to(device)\n            label = data[\"label\"].type(torch.uint8).to(device)\n            y_pred = model(img)\n            y_pred_label = torch.argmax(y_pred, dim=1)\n            val_labels.extend(label.cpu().detach())\n            val_preds.extend(y_pred_label.cpu().detach())\n            \n            loss = loss_func(y_pred, label)\n            val_cost += loss\n    val_cost = val_cost / (1+idx)\n    \n    print(\"-\"*30)\n    print(f\"Train Loss EPOCH {epoch+1}: {train_cost:.4f}\")\n    print(f\"Valid Loss EPOCH {epoch+1}: {val_cost:.4f}\")\n    print(f\"Train Accuracy EPOCH {epoch+1}: {sum(1 for x,y in zip(train_preds, train_labels) if x == y) / len(train_labels):.4f}\")\n    print(f\"Valid Accuracy EPOCH {epoch+1}: {sum(1 for x,y in zip(val_preds, val_labels) if x == y) / len(val_labels):.4f}\")\n    print(\"-\"*30)","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:18:14.461738Z","iopub.execute_input":"2024-01-11T08:18:14.462018Z","iopub.status.idle":"2024-01-11T08:48:56.062470Z","shell.execute_reply.started":"2024-01-11T08:18:14.461995Z","shell.execute_reply":"2024-01-11T08:48:56.061506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:48:56.063748Z","iopub.execute_input":"2024-01-11T08:48:56.064153Z","iopub.status.idle":"2024-01-11T08:48:56.119198Z","shell.execute_reply.started":"2024-01-11T08:48:56.064114Z","shell.execute_reply":"2024-01-11T08:48:56.118305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nids = []\nimgs = []\nmodel.eval()\nwith torch.no_grad():\n    for idx, sample in enumerate(tqdm(test_dataloader, position=0, leave=True)):\n        img = sample[\"image\"].to(device)\n        ids.extend([int(i)+1 for i in sample[\"index\"]])\n        \n        outputs = model(img)\n        \n        imgs.extend(img.detach().cpu())\n        labels.extend([int(i) for i in torch.argmax(outputs, dim=1)])","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:48:56.120337Z","iopub.execute_input":"2024-01-11T08:48:56.120629Z","iopub.status.idle":"2024-01-11T08:49:00.591416Z","shell.execute_reply.started":"2024-01-11T08:48:56.120603Z","shell.execute_reply":"2024-01-11T08:49:00.590502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nf, axarr = plt.subplots(2, 3)\ncounter = 0\nfor i in range(2):\n    for j in range(3):\n        axarr[i][j].imshow(imgs[counter].squeeze(), cmap=\"gray\")\n        axarr[i][j].set_title(f\"{labels[counter]}\")\n        counter += 1","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:49:00.592521Z","iopub.execute_input":"2024-01-11T08:49:00.592815Z","iopub.status.idle":"2024-01-11T08:49:01.509133Z","shell.execute_reply.started":"2024-01-11T08:49:00.592781Z","shell.execute_reply":"2024-01-11T08:49:01.508223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(list(zip(ids, labels)),\n               columns =[\"ImageId\", \"Label\"])\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-11T08:49:01.510293Z","iopub.execute_input":"2024-01-11T08:49:01.510634Z","iopub.status.idle":"2024-01-11T08:49:01.604316Z","shell.execute_reply.started":"2024-01-11T08:49:01.510606Z","shell.execute_reply":"2024-01-11T08:49:01.603403Z"},"trusted":true},"execution_count":null,"outputs":[]}]}