{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":21755,"databundleVersionId":1475600,"sourceType":"competition"}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-05T20:47:04.257657Z","iopub.execute_input":"2024-02-05T20:47:04.258016Z","iopub.status.idle":"2024-02-05T20:47:27.467932Z","shell.execute_reply.started":"2024-02-05T20:47:04.257988Z","shell.execute_reply":"2024-02-05T20:47:27.467015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Preparation\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.1307,), (0.3081,))\n])\n\ntrain_dataset = datasets.MNIST('./data', train=True, download=True, transform=transform)\ntest_dataset = datasets.MNIST('./data', train=False, transform=transform)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=64, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T20:47:50.614963Z","iopub.execute_input":"2024-02-05T20:47:50.615489Z","iopub.status.idle":"2024-02-05T20:47:51.731203Z","shell.execute_reply.started":"2024-02-05T20:47:50.615460Z","shell.execute_reply":"2024-02-05T20:47:51.730392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model Architecture\nclass TransformerClassifier(nn.Module):\n    def __init__(self):\n        super(TransformerClassifier, self).__init__()\n        self.transformer_encoder = nn.TransformerEncoder(\n            nn.TransformerEncoderLayer(d_model=28*28, nhead=8),\n            num_layers=6)\n        self.fc = nn.Linear(28*28, 10)\n        \n    def forward(self, x):\n        x = x.view(-1, 28*28)\n        x = self.transformer_encoder(x)\n        x = self.fc(x)\n        return x\n\nmodel = TransformerClassifier()","metadata":{"execution":{"iopub.status.busy":"2024-02-05T20:47:55.611600Z","iopub.execute_input":"2024-02-05T20:47:55.611962Z","iopub.status.idle":"2024-02-05T20:47:55.790292Z","shell.execute_reply.started":"2024-02-05T20:47:55.611934Z","shell.execute_reply":"2024-02-05T20:47:55.789291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\ndef train(model, train_loader, criterion, optimizer, num_epochs):\n    for epoch in range(num_epochs):\n        running_loss = 0.0\n        for i, (inputs, labels) in enumerate(train_loader):\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        print('Epoch [%d/%d], Loss: %.4f' % (epoch+1, num_epochs, running_loss/len(train_loader)))\n\ntrain(model, train_loader, criterion, optimizer, 10)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T20:48:01.049962Z","iopub.execute_input":"2024-02-05T20:48:01.050555Z","iopub.status.idle":"2024-02-05T21:41:52.883579Z","shell.execute_reply.started":"2024-02-05T20:48:01.050522Z","shell.execute_reply":"2024-02-05T21:41:52.882613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\ndef test(model, test_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for inputs, labels in test_loader:\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    print('Accuracy: %.2f %%' % (100 * correct / total))\n\ntest(model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T21:43:02.620307Z","iopub.execute_input":"2024-02-05T21:43:02.620701Z","iopub.status.idle":"2024-02-05T21:43:13.696820Z","shell.execute_reply.started":"2024-02-05T21:43:02.620672Z","shell.execute_reply":"2024-02-05T21:43:13.695832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}