{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":16017,"databundleVersionId":708189,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-25T13:34:43.679906Z","iopub.execute_input":"2024-03-25T13:34:43.681226Z","iopub.status.idle":"2024-03-25T13:34:44.793401Z","shell.execute_reply.started":"2024-03-25T13:34:43.681180Z","shell.execute_reply":"2024-03-25T13:34:44.792284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(r'/kaggle/input/Kannada-MNIST/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:34:47.009174Z","iopub.execute_input":"2024-03-25T13:34:47.009671Z","iopub.status.idle":"2024-03-25T13:34:52.237620Z","shell.execute_reply.started":"2024-03-25T13:34:47.009642Z","shell.execute_reply":"2024-03-25T13:34:52.236379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = train['label']\ntrain.drop('label', inplace = True, axis = \"columns\")\nprint(label.shape)\nprint(label.head())\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:34:53.366447Z","iopub.execute_input":"2024-03-25T13:34:53.366812Z","iopub.status.idle":"2024-03-25T13:34:53.528967Z","shell.execute_reply.started":"2024-03-25T13:34:53.366784Z","shell.execute_reply":"2024-03-25T13:34:53.527772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset, Dataset, DataLoader, Subset, random_split\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms, models\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torchvision.transforms import v2\n\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:34:56.491246Z","iopub.execute_input":"2024-03-25T13:34:56.491618Z","iopub.status.idle":"2024-03-25T13:35:04.461462Z","shell.execute_reply.started":"2024-03-25T13:34:56.491591Z","shell.execute_reply":"2024-03-25T13:35:04.460170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tensor = torch.tensor(train.to_numpy() / 255, dtype=torch.float32)\nlabel_tensor = torch.tensor(label.to_numpy(), dtype=torch.int64)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:04.463516Z","iopub.execute_input":"2024-03-25T13:35:04.463997Z","iopub.status.idle":"2024-03-25T13:35:04.823881Z","shell.execute_reply.started":"2024-03-25T13:35:04.463969Z","shell.execute_reply":"2024-03-25T13:35:04.822904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(4, 4))\nplt.imshow(train_tensor[99].reshape(28, 28), cmap=\"binary\")\nplt.title(f'Target label for this image: {label_tensor[99]}')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:11.139994Z","iopub.execute_input":"2024-03-25T13:35:11.140960Z","iopub.status.idle":"2024-03-25T13:35:11.401983Z","shell.execute_reply.started":"2024-03-25T13:35:11.140924Z","shell.execute_reply":"2024-03-25T13:35:11.400842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_tensor should be in [N, C, H, W] format for input to neural network\ntrain_tensor = train_tensor.reshape(train_tensor.shape[0], 28, 28)\nprint(train_tensor.shape)\ntrain_tensor = torch.unsqueeze(train_tensor, 1)\nprint(train_tensor.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:14.121759Z","iopub.execute_input":"2024-03-25T13:35:14.122161Z","iopub.status.idle":"2024-03-25T13:35:14.137079Z","shell.execute_reply.started":"2024-03-25T13:35:14.122129Z","shell.execute_reply":"2024-03-25T13:35:14.136004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create PyTorch Dataset using TensorDataset(input_features, labels):\nmnist_dataset = TensorDataset(train_tensor, label_tensor) ","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:14.665577Z","iopub.execute_input":"2024-03-25T13:35:14.666396Z","iopub.status.idle":"2024-03-25T13:35:14.670979Z","shell.execute_reply.started":"2024-03-25T13:35:14.666357Z","shell.execute_reply":"2024-03-25T13:35:14.669942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_set, validation_set = random_split(mnist_dataset, [50000, 10000], generator=torch.Generator().manual_seed(1))\n\nbatch_size = 256\ntorch.manual_seed(1)\n\ntrain_dl = DataLoader(training_set, batch_size, shuffle=True)\nvalid_dl = DataLoader(validation_set, batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:16.122004Z","iopub.execute_input":"2024-03-25T13:35:16.122384Z","iopub.status.idle":"2024-03-25T13:35:16.144593Z","shell.execute_reply.started":"2024-03-25T13:35:16.122358Z","shell.execute_reply":"2024-03-25T13:35:16.143526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Block(nn.Module):\n    \n    def __init__(self, in_channels, out_channels, identity_downsample=None, stride=1):\n        super(Block, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU()\n        self.identity_downsample = identity_downsample\n        \n    def forward(self, x):\n        identity = x\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.conv2(x)\n        x = self.bn2(x)\n        if self.identity_downsample is not None:\n            identity = self.identity_downsample(identity)\n        x += identity\n        x = self.relu(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:18.153872Z","iopub.execute_input":"2024-03-25T13:35:18.154287Z","iopub.status.idle":"2024-03-25T13:35:18.165873Z","shell.execute_reply.started":"2024-03-25T13:35:18.154256Z","shell.execute_reply":"2024-03-25T13:35:18.164690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNet_18(nn.Module):\n    \n    def __init__(self, image_channels, num_classes):\n        \n        super(ResNet_18, self).__init__()\n        self.in_channels = 64\n        self.conv1 = nn.Conv2d(image_channels, 64, kernel_size=7, stride=2, padding=3)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU()\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        \n        #resnet layers\n        self.layer1 = self.__make_layer(64, 64, stride=1)\n        self.layer2 = self.__make_layer(64, 128, stride=2)\n        self.layer3 = self.__make_layer(128, 256, stride=2)\n        self.layer4 = self.__make_layer(256, 512, stride=2)\n        \n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512, num_classes)\n        \n    def __make_layer(self, in_channels, out_channels, stride):\n        \n        identity_downsample = None\n        if stride != 1:\n            identity_downsample = self.identity_downsample(in_channels, out_channels)\n            \n        return nn.Sequential(\n            Block(in_channels, out_channels, identity_downsample=identity_downsample, stride=stride), \n            Block(out_channels, out_channels)\n        )\n        \n    def forward(self, x):\n        \n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n        \n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        \n        x = self.avgpool(x)\n        x = x.view(x.shape[0], -1)\n        x = self.fc(x)\n        return x \n    \n    def identity_downsample(self, in_channels, out_channels):\n        \n        return nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=2, padding=1), \n            nn.BatchNorm2d(out_channels)\n        )","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:35:21.286575Z","iopub.execute_input":"2024-03-25T13:35:21.287520Z","iopub.status.idle":"2024-03-25T13:35:21.302377Z","shell.execute_reply.started":"2024-03-25T13:35:21.287484Z","shell.execute_reply":"2024-03-25T13:35:21.301553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_ft = ResNet_18(1,10)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:36:15.782785Z","iopub.execute_input":"2024-03-25T13:36:15.783514Z","iopub.status.idle":"2024-03-25T13:36:15.900184Z","shell.execute_reply.started":"2024-03-25T13:36:15.783467Z","shell.execute_reply":"2024-03-25T13:36:15.899117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()\n\n# Observe that all parameters are being optimized\n# optimizer = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\noptimizer = torch.optim.Adam(model_ft.parameters(), lr=0.001)\n# Decay LR by a factor of 0.1 every 7 epochs\n#exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\nnum_epochs = 10","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:36:17.269844Z","iopub.execute_input":"2024-03-25T13:36:17.270239Z","iopub.status.idle":"2024-03-25T13:36:17.277442Z","shell.execute_reply.started":"2024-03-25T13:36:17.270208Z","shell.execute_reply":"2024-03-25T13:36:17.276112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, num_epochs, train_dl, valid_dl):\n\n    loss_hist_train = [0] * num_epochs\n    accuracy_hist_train = [0] * num_epochs\n    loss_hist_valid = [0] * num_epochs\n    accuracy_hist_valid = [0] * num_epochs\n    \n    for epoch in range(num_epochs):\n        \n        model.train()\n\n        for x_batch, y_batch in train_dl:\n\n            pred = model(x_batch)   # Get predicted values for this batch       \n            loss = loss_fn(pred, y_batch)   # Calculate the loss\n            loss.backward()   # Compute gradients\n            optimizer.step()   # Update parameters\n            optimizer.zero_grad()   # Reset grads to zero\n            #exp_lr_scheduler.step()\n            loss_hist_train[epoch] += loss.item()*y_batch.size(0)    # Total loss for the whole batch\n            \n            # Each image in batch has predicted class probabilities for classes 0 to 9.\n            # Take highest class probability (our predicted label) and check if it equals true label.\n            # Therefore is_correct = 1 if image prediction is correct, 0 otherwise.\n            \n            is_correct = (torch.argmax(pred, dim=1) == y_batch).float()   \n            \n            accuracy_hist_train[epoch] += is_correct.sum()    # Number of correctly predicted images in this batch\n\n        loss_hist_train[epoch] /= len(train_dl.dataset)    # After accumulating total losses from each batch, calculate loss per sample.\n        accuracy_hist_train[epoch] /= len(train_dl.dataset)   # After getting total number of correct preds across all batches, calculate accuracy per sample (so between 0 and 1). \n        \n        \n        \n        # Put model in eval mode for evaluating on validation set\n        model.eval()  \n        \n        with torch.no_grad():  # Disable gradient calculation, since we do not need this for getting validation set accuracy and loss\n            for x_batch, y_batch in valid_dl:\n\n                pred = model(x_batch)\n                loss = loss_fn(pred, y_batch)\n                loss_hist_valid[epoch] += loss.item()*y_batch.size(0)\n                is_correct = (torch.argmax(pred, dim=1) == y_batch).float()\n                accuracy_hist_valid[epoch] += is_correct.sum()\n                \n        loss_hist_valid[epoch] /= len(valid_dl.dataset)\n        accuracy_hist_valid[epoch] /= len(valid_dl.dataset)\n                \n        print(f'Epoch {epoch + 1}    Train Accuracy: {accuracy_hist_train[epoch]:.4f}    Validation Accuracy: {accuracy_hist_valid[epoch]:.4f}     Train Loss: {loss_hist_train[epoch]:.4f}    Validation Loss: {loss_hist_valid[epoch]:.4f} ')\n        \n    return loss_hist_train, loss_hist_valid, accuracy_hist_train, accuracy_hist_valid","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:36:19.801083Z","iopub.execute_input":"2024-03-25T13:36:19.801474Z","iopub.status.idle":"2024-03-25T13:36:19.815929Z","shell.execute_reply.started":"2024-03-25T13:36:19.801446Z","shell.execute_reply":"2024-03-25T13:36:19.814636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = train(model_ft, num_epochs, train_dl, valid_dl)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T11:45:13.864520Z","iopub.execute_input":"2024-03-25T11:45:13.864932Z","iopub.status.idle":"2024-03-25T12:41:56.522018Z","shell.execute_reply.started":"2024-03-25T11:45:13.864899Z","shell.execute_reply":"2024-03-25T12:41:56.521012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_custom = train(model_ft, num_epochs, train_dl, valid_dl)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T13:36:36.766368Z","iopub.execute_input":"2024-03-25T13:36:36.766749Z","iopub.status.idle":"2024-03-25T14:31:19.528358Z","shell.execute_reply.started":"2024-03-25T13:36:36.766701Z","shell.execute_reply":"2024-03-25T14:31:19.526856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare test\ntest = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\nsample = pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv')\nprint(test.shape)\nprint(sample.shape)\ntest.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-25T14:31:25.007188Z","iopub.execute_input":"2024-03-25T14:31:25.007992Z","iopub.status.idle":"2024-03-25T14:31:25.484964Z","shell.execute_reply.started":"2024-03-25T14:31:25.007944Z","shell.execute_reply":"2024-03-25T14:31:25.483524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = test['id']\ntest.drop('id', inplace = True, axis = \"columns\")\nprint(ids.head())\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T14:31:30.604178Z","iopub.execute_input":"2024-03-25T14:31:30.604585Z","iopub.status.idle":"2024-03-25T14:31:30.642065Z","shell.execute_reply.started":"2024-03-25T14:31:30.604555Z","shell.execute_reply":"2024-03-25T14:31:30.640628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest_tensor = torch.tensor(test.to_numpy() / 255, dtype=torch.float32)\ntest_tensor = test_tensor.reshape(test_tensor.shape[0], 28, 28)\ntest_tensor = torch.unsqueeze(test_tensor, 1)\ntest_tensor.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-25T14:31:34.937459Z","iopub.execute_input":"2024-03-25T14:31:34.937873Z","iopub.status.idle":"2024-03-25T14:31:34.965960Z","shell.execute_reply.started":"2024-03-25T14:31:34.937843Z","shell.execute_reply":"2024-03-25T14:31:34.964783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predicted_logits = model_ft(test_tensor)\ntest_predicted_logits.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-25T14:31:35.963863Z","iopub.execute_input":"2024-03-25T14:31:35.964294Z","iopub.status.idle":"2024-03-25T14:31:43.775749Z","shell.execute_reply.started":"2024-03-25T14:31:35.964263Z","shell.execute_reply":"2024-03-25T14:31:43.774436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Our NN outputs logits for each class.\n\nprint(test_predicted_logits[:5])\ntest_predicted_classes = torch.argmax(test_predicted_logits, dim=1)\nprint(test_predicted_classes[:5])\nprint(test_predicted_classes.shape)\ntest_predicted_classes.numpy()\n# test_predicted_classes.to('cpu').numpy()\n#image_ids = np.arange(1, len(test_predicted_classes)+1)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T14:31:43.777804Z","iopub.execute_input":"2024-03-25T14:31:43.779049Z","iopub.status.idle":"2024-03-25T14:31:43.807747Z","shell.execute_reply.started":"2024-03-25T14:31:43.779003Z","shell.execute_reply":"2024-03-25T14:31:43.806534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({\"ImageId\":ids, \"Label\":test_predicted_classes.to('cpu')})\ndf.to_csv(\"submission.csv\", index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T14:31:49.721008Z","iopub.execute_input":"2024-03-25T14:31:49.721473Z","iopub.status.idle":"2024-03-25T14:31:49.745683Z","shell.execute_reply.started":"2024-03-25T14:31:49.721436Z","shell.execute_reply":"2024-03-25T14:31:49.744571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}