{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-25T04:46:24.704767Z","iopub.execute_input":"2023-03-25T04:46:24.705320Z","iopub.status.idle":"2023-03-25T04:46:24.748239Z","shell.execute_reply.started":"2023-03-25T04:46:24.705262Z","shell.execute_reply":"2023-03-25T04:46:24.746605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision.datasets import MNIST\nfrom torchvision.transforms import ToTensor\nfrom torchvision.utils import make_grid\nimport torchvision.models as models\nfrom sklearn.model_selection import train_test_split\nimport warnings\n# Ignore all warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-03-25T05:54:39.465254Z","iopub.execute_input":"2023-03-25T05:54:39.466641Z","iopub.status.idle":"2023-03-25T05:54:39.907420Z","shell.execute_reply.started":"2023-03-25T05:54:39.466566Z","shell.execute_reply":"2023-03-25T05:54:39.905828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-03-25T05:54:39.943974Z","iopub.execute_input":"2023-03-25T05:54:39.944482Z","iopub.status.idle":"2023-03-25T05:54:39.952765Z","shell.execute_reply.started":"2023-03-25T05:54:39.944438Z","shell.execute_reply":"2023-03-25T05:54:39.951699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv',dtype=np.float32)\ndisplay(train.head())\nprint(f\"train shape: {train.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:16.799838Z","iopub.execute_input":"2023-03-25T06:03:16.801519Z","iopub.status.idle":"2023-03-25T06:03:19.716000Z","shell.execute_reply.started":"2023-03-25T06:03:16.801449Z","shell.execute_reply":"2023-03-25T06:03:19.714391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv',dtype=np.float32)\ndisplay(test.head())\nprint(f\"test shape: {test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:19.718622Z","iopub.execute_input":"2023-03-25T06:03:19.719002Z","iopub.status.idle":"2023-03-25T06:03:21.632582Z","shell.execute_reply.started":"2023-03-25T06:03:19.718968Z","shell.execute_reply":"2023-03-25T06:03:21.631390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train['label']\nx_train = train.loc[:, train.columns != 'label'].values","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:21.634157Z","iopub.execute_input":"2023-03-25T06:03:21.634604Z","iopub.status.idle":"2023-03-25T06:03:21.702944Z","shell.execute_reply.started":"2023-03-25T06:03:21.634558Z","shell.execute_reply":"2023-03-25T06:03:21.701587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:21.705684Z","iopub.execute_input":"2023-03-25T06:03:21.706097Z","iopub.status.idle":"2023-03-25T06:03:21.716348Z","shell.execute_reply.started":"2023-03-25T06:03:21.706057Z","shell.execute_reply":"2023-03-25T06:03:21.714662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train/255.0\nx_test = test/255.0\nprint(f\"x_train shape: {x_train.shape}\")\nprint(f\"x_test shape: {test.shape}\")\nx_train = x_train.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:21.718055Z","iopub.execute_input":"2023-03-25T06:03:21.718511Z","iopub.status.idle":"2023-03-25T06:03:21.816907Z","shell.execute_reply.started":"2023-03-25T06:03:21.718475Z","shell.execute_reply":"2023-03-25T06:03:21.815713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(15,4.5))\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    plt.imshow(x_train[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:21.818175Z","iopub.execute_input":"2023-03-25T06:03:21.818521Z","iopub.status.idle":"2023-03-25T06:03:23.374490Z","shell.execute_reply.started":"2023-03-25T06:03:21.818488Z","shell.execute_reply":"2023-03-25T06:03:23.373545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\n# x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)\n\nx_train = x_train.reshape((-1,1,28,28))\nx_test = x_test.values.reshape(-1,1,28,28)\n# x_train = np.stack((x_train,)*3, axis=1)\n# x_train = np.squeeze(x_train, axis=2)\nx_train.shape\n\n# print(f\"X_train shape: {x_train.shape}\")\n# print(f\"test shape: {x_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:23.375901Z","iopub.execute_input":"2023-03-25T06:03:23.377134Z","iopub.status.idle":"2023-03-25T06:03:23.386915Z","shell.execute_reply.started":"2023-03-25T06:03:23.377077Z","shell.execute_reply":"2023-03-25T06:03:23.385561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val,y_train, y_val = train_test_split(x_train,y_train, test_size = 0.2,random_state = 42) ","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:03:23.388545Z","iopub.execute_input":"2023-03-25T06:03:23.389020Z","iopub.status.idle":"2023-03-25T06:03:23.777270Z","shell.execute_reply.started":"2023-03-25T06:03:23.388971Z","shell.execute_reply":"2023-03-25T06:03:23.775887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_tensor = torch.tensor(x_train)\ny_train_tensor = torch.tensor(y_train.values).type(torch.LongTensor)\nx_val_tensor = torch.tensor(x_val)\ny_val_tensor = torch.tensor(y_val.values).type(torch.LongTensor)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:05:21.780339Z","iopub.execute_input":"2023-03-25T06:05:21.780893Z","iopub.status.idle":"2023-03-25T06:05:21.864478Z","shell.execute_reply.started":"2023-03-25T06:05:21.780831Z","shell.execute_reply":"2023-03-25T06:05:21.863209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 32\nepochs = 20\nLR = 1e-3","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:08:20.242778Z","iopub.execute_input":"2023-03-25T06:08:20.244223Z","iopub.status.idle":"2023-03-25T06:08:20.250603Z","shell.execute_reply.started":"2023-03-25T06:08:20.244133Z","shell.execute_reply":"2023-03-25T06:08:20.249150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNISTDataset(Dataset):\n    def __init__(self, features, labels):\n        self.features = features\n        self.labels = labels\n        \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, idx):\n        return self.features[idx], self.labels[idx]","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:09:40.251181Z","iopub.execute_input":"2023-03-25T06:09:40.251709Z","iopub.status.idle":"2023-03-25T06:09:40.260666Z","shell.execute_reply.started":"2023-03-25T06:09:40.251664Z","shell.execute_reply":"2023-03-25T06:09:40.259001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = MNISTDataset(x_train_tensor, y_train_tensor)\nvalidation_dataset = MNISTDataset(x_val_tensor, y_val_tensor)\n\n# Create a dataloader object from the dataset\nbatch_size = 256\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nvalidation_dataloader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T07:13:35.543353Z","iopub.execute_input":"2023-03-25T07:13:35.543927Z","iopub.status.idle":"2023-03-25T07:13:35.551804Z","shell.execute_reply.started":"2023-03-25T07:13:35.543867Z","shell.execute_reply":"2023-03-25T07:13:35.550369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"next(iter(train_dataloader))[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-03-25T07:13:37.644823Z","iopub.execute_input":"2023-03-25T07:13:37.645350Z","iopub.status.idle":"2023-03-25T07:13:37.665929Z","shell.execute_reply.started":"2023-03-25T07:13:37.645307Z","shell.execute_reply":"2023-03-25T07:13:37.663994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class ConvNet(nn.Module):\n#     def __init__(self):\n#         super(ConvNet, self).__init__()\n#         self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1) #256x1x28x28\n#         self.relu1 = nn.ReLU()\n#         self.norm1 = nn.BatchNorm2d(32)                                   #256x32x28x28\n        \n#         self.conv2 = nn.Conv2d(32, 32, kernel_size=3, padding=1)\n#         self.relu2 = nn.ReLU()\n#         self.norm2 = nn.BatchNorm2d(32)  \n#         self.drop1 = nn.Dropout2d(0.4)                                    #256x32x28x28\n#         #self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2),                \n        \n#         self.conv3 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n#         self.relu3 = nn.ReLU()\n#         self.norm3= nn.BatchNorm2d(64)                                    #256x64x28x28\n#         #self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2),\n        \n#         self.conv4 = nn.Conv2d(64, 64, kernel_size=3, padding=1)\n#         self.relu4 = nn.ReLU()\n#         self.norm4 = nn.BatchNorm2d(64)\n#         #self.drop1 = nn.Dropout2d(0.4)                                     #256x64x14x14\n        \n#         self.conv5 = nn.Conv2d(64, 64, kernel_size=5, padding=2, stride=2)\n#         self.relu5 = nn.ReLU()\n#         self.norm5 = nn.BatchNorm2d(64) \n#         self.drop2 = nn.Dropout2d(0.4)                                  #256x64x14x14\n#         #self.pool5 = nn.MaxPool2d(kernel_size=2, stride=2), \n        \n#         self.conv6 = nn.Conv2d(64, 128, kernel_size=4)\n#         self.relu6 = nn.ReLU()\n#         self.norm6 = nn.BatchNorm2d(128)                                     #256x128x7x7\n              \n        \n#         self.fc1 = nn.Linear(2048, 10)\n# #         self.relu7 = nn.ReLU()\n# #         self.fc2 = nn.Linear(128, 10)\n    \n# #         self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)\n# #         self.relu2 = nn.ReLU()\n# #         self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n# #         self.fc1 = nn.Linear(64*7*7, 128)\n# #         self.relu3 = nn.ReLU()\n# #         self.fc2 = nn.Linear(128, 10)\n\n#     def forward(self, x):\n#         x = self.conv1(x)\n#         x = self.relu1(x)\n#         x = self.norm1(x)\n        \n#         x = self.conv2(x)\n#         x = self.relu2(x)\n#         x = self.norm2(x)\n#         x = self.drop1(x)\n        \n#         x = self.conv3(x)\n#         x = self.relu3(x)\n#         x = self.norm3(x)\n        \n#         x = self.conv4(x)\n#         x = self.relu4(x)\n#         x = self.norm4(x)\n#         #x = self.pool4(x)\n        \n#         x = self.conv5(x)\n#         x = self.relu5(x)\n#         x = self.norm5(x)\n#         x = self.drop2(x)\n        \n#         x = self.conv6(x)\n#         x = self.relu6(x)\n#         x = self.norm6(x)\n#         #x = self.pool6(x)\n        \n#         x = x.view(x.size(0), -1) # flatten the tensor for the fully connected layers\n#         x = self.fc1(x)\n# #         x = self.relu7(x)\n# #         x = self.fc2(x)\n#         return x\n    \n\n# model = ConvNet().to(device)\n#print(model)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T07:09:26.878190Z","iopub.execute_input":"2023-03-25T07:09:26.878893Z","iopub.status.idle":"2023-03-25T07:09:26.912672Z","shell.execute_reply.started":"2023-03-25T07:09:26.878814Z","shell.execute_reply":"2023-03-25T07:09:26.910963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvNet(nn.Module):\n    def __init__(self):\n        super(ConvNet, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(64*7*7, 128)\n        self.relu3 = nn.ReLU()\n        self.fc2 = nn.Linear(128, 10)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.relu1(x)\n        x = self.pool1(x)\n        x = self.conv2(x)\n        x = self.relu2(x)\n        x = self.pool2(x)\n        x = x.view(x.size(0), -1) # flatten the tensor for the fully connected layers\n        x = self.fc1(x)\n        x = self.relu3(x)\n        x = self.fc2(x)\n        return x\n    \n# Get cpu or gpu device for training.\ndevice = \"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\"\nprint(f\"Using {device} device\")\n\nmodel = ConvNet().to(device)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T07:13:42.494178Z","iopub.execute_input":"2023-03-25T07:13:42.494706Z","iopub.status.idle":"2023-03-25T07:13:42.519146Z","shell.execute_reply.started":"2023-03-25T07:13:42.494650Z","shell.execute_reply":"2023-03-25T07:13:42.517321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()\n# define optimiser\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n\n# train\ndef train(dataloader, model, loss_fn, optimizer):\n    size = len(dataloader.dataset)\n    model.train()\n    for batch, (X, y) in enumerate(dataloader):\n        X, y = X.to(device), y.to(device)\n\n        # Compute prediction error\n        pred = model(X)\n        loss = loss_fn(pred, y)\n\n        # Backpropagation\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        if batch % 100 == 0:\n            loss, current = loss.item(), (batch + 1) * len(X)\n            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n            \n# validation\ndef validate(dataloader, model, loss_fn):\n    size = len(dataloader.dataset)\n    num_batches = len(dataloader)\n    model.eval()\n    validation_loss, correct = 0, 0\n    with torch.no_grad():\n        for X, y in dataloader:\n            X, y = X.to(device), y.to(device)\n            pred = model(X)\n            validation_loss += loss_fn(pred, y).item()\n            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n    validation_loss /= num_batches\n    correct /= size\n    validation_losses.append(validation_loss)\n    accuracy.append(correct)\n    print(f\"Validation Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {validation_loss:>8f} \\n\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T07:13:58.624066Z","iopub.execute_input":"2023-03-25T07:13:58.625599Z","iopub.status.idle":"2023-03-25T07:13:58.644030Z","shell.execute_reply.started":"2023-03-25T07:13:58.625502Z","shell.execute_reply":"2023-03-25T07:13:58.641938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10\nvalidation_losses = []\naccuracy = []\n\nfor t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train(train_dataloader, model, loss_fn, optimizer)\n    validate(validation_dataloader, model, loss_fn)\nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T07:16:04.648639Z","iopub.execute_input":"2023-03-25T07:16:04.649174Z","iopub.status.idle":"2023-03-25T07:20:01.994985Z","shell.execute_reply.started":"2023-03-25T07:16:04.649131Z","shell.execute_reply":"2023-03-25T07:20:01.993985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}