{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-30T14:56:26.103290Z","iopub.execute_input":"2023-08-30T14:56:26.103648Z","iopub.status.idle":"2023-08-30T14:56:26.121256Z","shell.execute_reply.started":"2023-08-30T14:56:26.103618Z","shell.execute_reply":"2023-08-30T14:56:26.120270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ndevice = (\n    \"cuda\"\n    if torch.cuda.is_available()\n    else \"mps\"\n    if torch.backends.mps.is_available()\n    else \"cpu\"\n)\nprint(f\"Using {device} device\")","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:27.026065Z","iopub.execute_input":"2023-08-30T14:56:27.026433Z","iopub.status.idle":"2023-08-30T14:56:30.016517Z","shell.execute_reply.started":"2023-08-30T14:56:27.026403Z","shell.execute_reply":"2023-08-30T14:56:30.015366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:31.696588Z","iopub.execute_input":"2023-08-30T14:56:31.697160Z","iopub.status.idle":"2023-08-30T14:56:31.966721Z","shell.execute_reply.started":"2023-08-30T14:56:31.697127Z","shell.execute_reply":"2023-08-30T14:56:31.965766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nimport torchvision","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:34.296608Z","iopub.execute_input":"2023-08-30T14:56:34.297579Z","iopub.status.idle":"2023-08-30T14:56:34.302895Z","shell.execute_reply.started":"2023-08-30T14:56:34.297535Z","shell.execute_reply":"2023-08-30T14:56:34.301857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:36.393032Z","iopub.execute_input":"2023-08-30T14:56:36.393783Z","iopub.status.idle":"2023-08-30T14:56:36.399050Z","shell.execute_reply.started":"2023-08-30T14:56:36.393737Z","shell.execute_reply":"2023-08-30T14:56:36.398068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/Kannada-MNIST/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:36.401262Z","iopub.execute_input":"2023-08-30T14:56:36.401938Z","iopub.status.idle":"2023-08-30T14:56:41.110937Z","shell.execute_reply.started":"2023-08-30T14:56:36.401905Z","shell.execute_reply":"2023-08-30T14:56:41.109684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:41.113102Z","iopub.execute_input":"2023-08-30T14:56:41.113470Z","iopub.status.idle":"2023-08-30T14:56:41.490370Z","shell.execute_reply.started":"2023-08-30T14:56:41.113443Z","shell.execute_reply":"2023-08-30T14:56:41.489333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val = pd.read_csv(\"/kaggle/input/Kannada-MNIST/Dig-MNIST.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:09:21.193503Z","iopub.execute_input":"2023-08-30T15:09:21.193945Z","iopub.status.idle":"2023-08-30T15:09:22.480002Z","shell.execute_reply.started":"2023-08-30T15:09:21.193908Z","shell.execute_reply":"2023-08-30T15:09:22.478863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_label = train[\"label\"]\ntr_pixel_vals = train.drop(\"label\",axis = 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:41.491826Z","iopub.execute_input":"2023-08-30T14:56:41.492868Z","iopub.status.idle":"2023-08-30T14:56:41.622454Z","shell.execute_reply.started":"2023-08-30T14:56:41.492839Z","shell.execute_reply":"2023-08-30T14:56:41.621305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tst_label = test[\"id\"]\ntst_pixel_vals = test.drop(\"id\", axis = 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:41.625130Z","iopub.execute_input":"2023-08-30T14:56:41.625687Z","iopub.status.idle":"2023-08-30T14:56:41.640157Z","shell.execute_reply.started":"2023-08-30T14:56:41.625652Z","shell.execute_reply":"2023-08-30T14:56:41.638901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"v_label = val['label']\nv_pixel_vals = val.drop(\"label\",axis = 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:10:16.098180Z","iopub.execute_input":"2023-08-30T15:10:16.099079Z","iopub.status.idle":"2023-08-30T15:10:16.126153Z","shell.execute_reply.started":"2023-08-30T15:10:16.099035Z","shell.execute_reply":"2023-08-30T15:10:16.124867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nimport matplotlib.pyplot as plt\n\nclass CustomImageDataset(Dataset):\n    def __init__(self, labels, pixel_vals, transform=None, target_transform=None):\n        self.labels = labels\n        self.pixels = pixel_vals\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        pixel_values = self.pixels[idx].astype(float).reshape(28, 28)  # Convert pixel values to float\n        label = self.labels[idx]  # Assuming the label is in the second column\n\n        if self.transform:\n            pixel_values = self.transform(pixel_values)\n\n        if self.target_transform:\n            label = self.target_transform(label)\n\n        return pixel_values, label\n\nTrain = CustomImageDataset(tr_label, tr_pixel_vals.values)\nTest = CustomImageDataset(tst_label, tst_pixel_vals.values)\nVal = CustomImageDataset(v_label, v_pixel_vals.values)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:10:16.930025Z","iopub.execute_input":"2023-08-30T15:10:16.930396Z","iopub.status.idle":"2023-08-30T15:10:16.940096Z","shell.execute_reply.started":"2023-08-30T15:10:16.930366Z","shell.execute_reply":"2023-08-30T15:10:16.938953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:41.652185Z","iopub.execute_input":"2023-08-30T14:56:41.652832Z","iopub.status.idle":"2023-08-30T14:56:41.669821Z","shell.execute_reply.started":"2023-08-30T14:56:41.652794Z","shell.execute_reply":"2023-08-30T14:56:41.668789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Test","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:42.457277Z","iopub.execute_input":"2023-08-30T14:56:42.457974Z","iopub.status.idle":"2023-08-30T14:56:42.464702Z","shell.execute_reply.started":"2023-08-30T14:56:42.457939Z","shell.execute_reply":"2023-08-30T14:56:42.463709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Val","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:10:21.618282Z","iopub.execute_input":"2023-08-30T15:10:21.619177Z","iopub.status.idle":"2023-08-30T15:10:21.626643Z","shell.execute_reply.started":"2023-08-30T15:10:21.619142Z","shell.execute_reply":"2023-08-30T15:10:21.625488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 3, figsize=(8, 8))\n\nfor idx in range(9):\n    pixel_values, label = Train[idx]\n    image = pixel_values\n\n    row_idx = idx // 3\n    col_idx = idx % 3\n    axes[row_idx, col_idx].imshow(image, cmap='gray')\n    axes[row_idx, col_idx].set_title(f\"Label: {label}\")\n    axes[row_idx, col_idx].axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:44.401282Z","iopub.execute_input":"2023-08-30T14:56:44.401643Z","iopub.status.idle":"2023-08-30T14:56:45.524733Z","shell.execute_reply.started":"2023-08-30T14:56:44.401612Z","shell.execute_reply":"2023-08-30T14:56:45.523845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\n\ntrain_dataloader = DataLoader(Train, batch_size=64, shuffle=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:50.395713Z","iopub.execute_input":"2023-08-30T14:56:50.396074Z","iopub.status.idle":"2023-08-30T14:56:50.402003Z","shell.execute_reply.started":"2023-08-30T14:56:50.396043Z","shell.execute_reply":"2023-08-30T14:56:50.400929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(Test, batch_size=64, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:56:52.760248Z","iopub.execute_input":"2023-08-30T14:56:52.760626Z","iopub.status.idle":"2023-08-30T14:56:52.765500Z","shell.execute_reply.started":"2023-08-30T14:56:52.760598Z","shell.execute_reply":"2023-08-30T14:56:52.764493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_dataloader = DataLoader(Val, batch_size = 64, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:10:31.656279Z","iopub.execute_input":"2023-08-30T15:10:31.656711Z","iopub.status.idle":"2023-08-30T15:10:31.662575Z","shell.execute_reply.started":"2023-08-30T15:10:31.656674Z","shell.execute_reply":"2023-08-30T15:10:31.661652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:57:03.386277Z","iopub.execute_input":"2023-08-30T14:57:03.386628Z","iopub.status.idle":"2023-08-30T14:57:03.393284Z","shell.execute_reply.started":"2023-08-30T14:57:03.386600Z","shell.execute_reply":"2023-08-30T14:57:03.392272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display image and label.\ntrain_features, train_labels = next(iter(train_dataloader))\nprint(f\"Feature batch shape: {train_features.size()}\")\nprint(f\"Labels batch shape: {train_labels.size()}\")\nimg = train_features[0].squeeze()\nlabel = train_labels[0]\nplt.imshow(img, cmap=\"gray\")\nplt.show()\nprint(f\"Label: {label}\")","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:57:05.265810Z","iopub.execute_input":"2023-08-30T14:57:05.266166Z","iopub.status.idle":"2023-08-30T14:57:05.554336Z","shell.execute_reply.started":"2023-08-30T14:57:05.266136Z","shell.execute_reply":"2023-08-30T14:57:05.553402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader","metadata":{"execution":{"iopub.status.busy":"2023-08-30T14:57:07.081321Z","iopub.execute_input":"2023-08-30T14:57:07.082035Z","iopub.status.idle":"2023-08-30T14:57:07.089339Z","shell.execute_reply.started":"2023-08-30T14:57:07.082003Z","shell.execute_reply":"2023-08-30T14:57:07.088274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training**","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport matplotlib.pyplot as plt\nfrom torchvision import transforms\n\n# Define the neural network architecture\nclass NeuralNetwork(nn.Module):\n    def __init__(self):\n        super(NeuralNetwork, self).__init__()\n        self.flatten = nn.Flatten()\n        self.linear_relu_stack = nn.Sequential(\n            nn.Linear(28*28, 512),\n            nn.ReLU(),\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Linear(256, 64),\n            nn.ReLU(),\n            nn.Linear(64,10)\n        )\n\n    def forward(self, x):\n        x = self.flatten(x)\n        x = x.type(self.linear_relu_stack[0].weight.dtype)\n        logits = self.linear_relu_stack(x)\n        return logits\n\n# Instantiate the model\nmodel = NeuralNetwork()\n\n# Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Lists to store loss values\ntrain_losses = []\nval_losses = []\n\n# Training loop\nnum_epochs = 10\nfor epoch in range(num_epochs):\n    model.train()\n    train_loss = 0.0\n    correct_train = 0\n    total_train = 0\n\n    for inputs, labels in train_dataloader:\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        train_loss += loss.item()\n        _, predicted = torch.max(outputs.data, 1)\n        total_train += labels.size(0)\n        correct_train += (predicted == labels).sum().item()\n\n    train_accuracy = 100 * correct_train / total_train\n    print(f\"Epoch [{epoch+1}/{num_epochs}] - Train Loss: {train_loss:.4f}, Train Accuracy: {train_accuracy:.2f}%\")\n\n    # Validation loop\n    model.eval()\n    val_loss = 0.0\n    correct_val = 0\n    total_val = 0\n\n    with torch.no_grad():\n        for inputs, labels in val_dataloader:\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total_val += labels.size(0)\n            correct_val += (predicted == labels).sum().item()\n\n    val_accuracy = 100 * correct_val / total_val\n    print(f\"Epoch [{epoch+1}/{num_epochs}] - Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%\")\n    \n    # Append loss values to the lists\n    train_losses.append(train_loss)\n    val_losses.append(val_loss)\n\n# Plotting the loss graph\nplt.figure()\nplt.plot(range(1, num_epochs+1), train_losses, label=\"Train Loss\")\nplt.plot(range(1, num_epochs+1), val_losses, label=\"Validation Loss\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Train Loss vs Validation Loss\")\nplt.legend()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:15:14.534420Z","iopub.execute_input":"2023-08-30T15:15:14.534774Z","iopub.status.idle":"2023-08-30T15:16:59.857813Z","shell.execute_reply.started":"2023-08-30T15:15:14.534745Z","shell.execute_reply":"2023-08-30T15:16:59.856197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Adding Regularization","metadata":{}},{"cell_type":"code","source":"import torch.nn.functional as F\n\n# Define the neural network architecture with L2 regularization\nclass NeuralNetwork(nn.Module):\n    def __init__(self):\n        super(NeuralNetwork, self).__init__()\n        self.flatten = nn.Flatten()\n        self.linear_relu_stack = nn.Sequential(\n            nn.Linear(28*28, 512),\n            nn.ReLU(),\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Linear(256, 64),\n            nn.ReLU(),\n            nn.Linear(64, 10)\n        )\n\n    def forward(self, x):\n        x = self.flatten(x)\n        x = x.type(self.linear_relu_stack[0].weight.dtype)\n        logits = self.linear_relu_stack(x)\n        return logits\n\n# Instantiate the model\nmodel = NeuralNetwork()\n\n# Define loss function and optimizer with weight decay (L2 regularization)\nweight_decay = 1e-5  # You can adjust this value\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=weight_decay)\n\n# Training loop with L2 regularization\nnum_epochs = 10\nfor epoch in range(num_epochs):\n    model.train()\n    train_loss = 0.0\n    correct_train = 0\n    total_train = 0\n\n    for inputs, labels in train_dataloader:\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        \n        # Add L2 regularization term to the loss\n        l2_reg = sum(p.norm(2) for p in model.parameters())\n        loss += weight_decay * l2_reg\n        \n        loss.backward()\n        optimizer.step()\n\n        train_loss += loss.item()\n        _, predicted = torch.max(outputs.data, 1)\n        total_train += labels.size(0)\n        correct_train += (predicted == labels).sum().item()\n\n    train_accuracy = 100 * correct_train / total_train\n    print(f\"Epoch [{epoch+1}/{num_epochs}] - Train Loss: {train_loss:.4f}, Train Accuracy: {train_accuracy:.2f}%\")\n\n    # Validation loop\n    model.eval()\n    val_loss = 0.0\n    correct_val = 0\n    total_val = 0\n\n    with torch.no_grad():\n        for inputs, labels in val_dataloader:\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total_val += labels.size(0)\n            correct_val += (predicted == labels).sum().item()\n\n    val_accuracy = 100 * correct_val / total_val\n    print(f\"Epoch [{epoch+1}/{num_epochs}] - Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%\")\n","metadata":{"execution":{"iopub.status.busy":"2023-08-30T15:19:55.101081Z","iopub.execute_input":"2023-08-30T15:19:55.101756Z","iopub.status.idle":"2023-08-30T15:22:30.296261Z","shell.execute_reply.started":"2023-08-30T15:19:55.101721Z","shell.execute_reply":"2023-08-30T15:22:30.294282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}