{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-04-05T09:59:47.401206Z","iopub.execute_input":"2022-04-05T09:59:47.401469Z","iopub.status.idle":"2022-04-05T09:59:47.415816Z","shell.execute_reply.started":"2022-04-05T09:59:47.401439Z","shell.execute_reply":"2022-04-05T09:59:47.415053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports - loading the data\n\nI used this notebook:\nhttps://www.kaggle.com/code/gabmars/pytorch-lenet5","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch import optim\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nimport torchmetrics\n#from torchsummary import summary\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device) # Might as well find out! ","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:47.435321Z","iopub.execute_input":"2022-04-05T09:59:47.435506Z","iopub.status.idle":"2022-04-05T09:59:47.446625Z","shell.execute_reply.started":"2022-04-05T09:59:47.435484Z","shell.execute_reply":"2022-04-05T09:59:47.445843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:47.47421Z","iopub.execute_input":"2022-04-05T09:59:47.474482Z","iopub.status.idle":"2022-04-05T09:59:50.945209Z","shell.execute_reply.started":"2022-04-05T09:59:47.474456Z","shell.execute_reply":"2022-04-05T09:59:50.944458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_data = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:50.949891Z","iopub.execute_input":"2022-04-05T09:59:50.951919Z","iopub.status.idle":"2022-04-05T09:59:51.51889Z","shell.execute_reply.started":"2022-04-05T09:59:50.951881Z","shell.execute_reply":"2022-04-05T09:59:51.518137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.522338Z","iopub.execute_input":"2022-04-05T09:59:51.52254Z","iopub.status.idle":"2022-04-05T09:59:51.527387Z","shell.execute_reply.started":"2022-04-05T09:59:51.522516Z","shell.execute_reply":"2022-04-05T09:59:51.526728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.529577Z","iopub.execute_input":"2022-04-05T09:59:51.53017Z","iopub.status.idle":"2022-04-05T09:59:51.552205Z","shell.execute_reply.started":"2022-04-05T09:59:51.529957Z","shell.execute_reply":"2022-04-05T09:59:51.551567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.553302Z","iopub.execute_input":"2022-04-05T09:59:51.553753Z","iopub.status.idle":"2022-04-05T09:59:51.560686Z","shell.execute_reply.started":"2022-04-05T09:59:51.553718Z","shell.execute_reply":"2022-04-05T09:59:51.559648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.562504Z","iopub.execute_input":"2022-04-05T09:59:51.562836Z","iopub.status.idle":"2022-04-05T09:59:51.579155Z","shell.execute_reply.started":"2022-04-05T09:59:51.562801Z","shell.execute_reply":"2022-04-05T09:59:51.578537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the data into features and target label\nX, y = (train_data.drop(['label'], axis=1), train_data.label)\nX_valid, y_valid = (valid_data.drop(['label'], axis=1), valid_data.label)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.580411Z","iopub.execute_input":"2022-04-05T09:59:51.580886Z","iopub.status.idle":"2022-04-05T09:59:51.708326Z","shell.execute_reply.started":"2022-04-05T09:59:51.580848Z","shell.execute_reply":"2022-04-05T09:59:51.707599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class KannadaDataSet(torch.utils.data.Dataset):\n    def __init__(self, images, labels, transforms = None):\n        self.images = images\n        self.labels = labels\n        self.transforms = transforms\n         \n    def __len__(self):\n        return self.images.shape[0]\n    \n    def __getitem__(self, i):\n        data = np.array(self.images.iloc[i,:]).astype(np.uint8).reshape(28,28,1)\n        \n        if self.transforms:\n            data = self.transforms(data)\n            \n        return (data, self.labels[i])","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.709621Z","iopub.execute_input":"2022-04-05T09:59:51.709896Z","iopub.status.idle":"2022-04-05T09:59:51.716862Z","shell.execute_reply.started":"2022-04-05T09:59:51.709863Z","shell.execute_reply":"2022-04-05T09:59:51.7162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose(([\n    transforms.ToTensor(),\n]))","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.717885Z","iopub.execute_input":"2022-04-05T09:59:51.718481Z","iopub.status.idle":"2022-04-05T09:59:51.726093Z","shell.execute_reply.started":"2022-04-05T09:59:51.718446Z","shell.execute_reply":"2022-04-05T09:59:51.725331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape, type(X)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.729536Z","iopub.execute_input":"2022-04-05T09:59:51.729739Z","iopub.status.idle":"2022-04-05T09:59:51.737215Z","shell.execute_reply.started":"2022-04-05T09:59:51.729711Z","shell.execute_reply":"2022-04-05T09:59:51.73654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[1000:2000]","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.738279Z","iopub.execute_input":"2022-04-05T09:59:51.73862Z","iopub.status.idle":"2022-04-05T09:59:51.761368Z","shell.execute_reply.started":"2022-04-05T09:59:51.738587Z","shell.execute_reply":"2022-04-05T09:59:51.76067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = KannadaDataSet(X, y, transform)\nvalid_set = KannadaDataSet(X_valid, y_valid, transform)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.762595Z","iopub.execute_input":"2022-04-05T09:59:51.762864Z","iopub.status.idle":"2022-04-05T09:59:51.766915Z","shell.execute_reply.started":"2022-04-05T09:59:51.762833Z","shell.execute_reply":"2022-04-05T09:59:51.766093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(43)\ntrain_loader = torch.utils.data.DataLoader(train_set, \n                                           batch_size=100, \n                                           shuffle=True)\n\nvalid_loader = torch.utils.data.DataLoader(valid_set,\n                                          batch_size=100, \n                                          shuffle=False)\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.768417Z","iopub.execute_input":"2022-04-05T09:59:51.76866Z","iopub.status.idle":"2022-04-05T09:59:51.775798Z","shell.execute_reply.started":"2022-04-05T09:59:51.768628Z","shell.execute_reply":"2022-04-05T09:59:51.775082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(iter(train_loader))\nplt.imshow(torchvision.utils.make_grid(images)[0,:,:], cmap='gray')\nprint(labels)\nimages[0,:,:].max()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.777093Z","iopub.execute_input":"2022-04-05T09:59:51.777363Z","iopub.status.idle":"2022-04-05T09:59:51.991583Z","shell.execute_reply.started":"2022-04-05T09:59:51.777311Z","shell.execute_reply":"2022-04-05T09:59:51.990905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"import time\ndef train(train_loader,valid_loader,model,optimizer,criterion,num_epochs=10,lr_scheduler=None):\n    \"\"\"\n    Function for training\n    \n    train_loader : training dataset loader\n    valid_loader : validation dataset loader\n    optimizer : optimizer algorithm\n    criterion : loss function\n    num_epochs : number of epochs\n    \n    The function returns (train_loss_list, train_acc_list, valid_loss_list, valid_acc_list), where\n    train_loss_list : average loss on mini-batches in the training set\n    train_acc_list : training accuracy\n    valid_loss_list : average loss on mini-batches in the validation set\n    valid_acc_list : average validation accuracy\n    \"\"\"\n    \n    train_loss_list = []\n    valid_loss_list = []\n    \n    train_acc_list = []\n    valid_acc_list = []\n    \n\n    epoch = 0\n    start = time.time()\n    \n    while epoch < num_epochs:\n        epoch+=1\n            \n        train_loss_temp = []\n        train_acc_temp = 0.0\n        train_total_samples = 0\n    \n        # training phase\n        for i, data in enumerate(train_loader, 0):\n\n            inputs, labels = data[0].to(device), data[1].to(device)\n\n            optimizer.zero_grad()\n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            # training loss\n            train_loss_temp.append(loss.item())\n        \n            # training accuracy\n            _, predicted = torch.max(outputs.data, 1)\n            train_acc_temp += (predicted == labels).sum().item()\n            train_total_samples += labels.size(0)\n        \n        train_loss_list.append(np.mean(train_loss_temp))\n        train_acc_list.append(train_acc_temp/train_total_samples)\n    \n        # validation set\n        with torch.no_grad():\n            valid_loss_temp = []\n            valid_acc_temp = 0.0\n            valid_total_samples = 0\n            model.eval()\n        \n            for v_i, v_data in enumerate(valid_loader,0):\n                v_inputs, v_labels = v_data[0].to(device), v_data[1].to(device)\n                v_outputs = model(v_inputs)\n                v_loss = criterion(v_outputs,v_labels)\n            \n                valid_loss_temp.append(v_loss.item())\n                _, predicted = torch.max(v_outputs.data,1)\n                \n            \n                valid_total_samples += v_labels.size(0)\n                valid_acc_temp += (predicted == v_labels).sum().item()\n            \n            valid_loss_list.append(np.mean(valid_loss_temp))\n            valid_acc_list.append(valid_acc_temp/valid_total_samples)\n            \n       \n\n        model.train()   \n        \n        # Scheduler\n        if lr_scheduler:\n            lr_scheduler.step()\n            \n        end = time.time()\n    \n        # Displaying the results\n        print(f\"[{epoch}/{num_epochs}] | TE: {train_loss_list[-1]:.6f} TA: {train_acc_list[-1]:.6f} | VE {valid_loss_list[-1]:.6f} VA: {valid_acc_list[-1]:.6f} | ET: {end-start:.2f}s\")\n\n    result_dict = {\"training loss\": train_loss_list, \n                   \"training acc\": train_acc_list,\n                   \"validation loss\": valid_loss_list, \n                   \"validation acc\": valid_acc_list\n                  }\n        \n    return result_dict","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:51.992901Z","iopub.execute_input":"2022-04-05T09:59:51.993317Z","iopub.status.idle":"2022-04-05T09:59:52.008501Z","shell.execute_reply.started":"2022-04-05T09:59:51.993279Z","shell.execute_reply":"2022-04-05T09:59:52.007689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def annot_max(x,y, ax=None):\n    xmax = x[np.argmax(y)]\n    ymax = y.max()\n    text= \"epoch: {}, valid acc: {:.3f}\".format(xmax, ymax)\n    if not ax:\n        ax=plt.gca()\n    bbox_props = dict(boxstyle=\"square,pad=0.3\", fc=\"w\", ec=\"k\", lw=0.72)\n    arrowprops=dict(arrowstyle=\"->\")#,connectionstyle=\"angle,angleA=0,angleB=60\")\n    kw = dict(xycoords='data',textcoords=\"axes fraction\",\n              arrowprops=arrowprops, bbox=bbox_props, ha=\"right\", va=\"top\")\n    ax.annotate(text, xy=(xmax, ymax), xytext=(0.94,0.96), **kw)\n\ndef plot_acc_loss(result_dict):\n    \"\"\"\n    Plotting function\n    \n    result_dict : containing the results from the `train` function\n    \"\"\"\n    \n    train_acc_list = result_dict[\"training acc\"]\n    train_loss_list = result_dict[\"training loss\"]\n    valid_acc_list = result_dict[\"validation acc\"]\n    valid_loss_list = result_dict[\"validation loss\"]\n    \n    \n    fig, ax = plt.subplots(1, 2, figsize = (8, 4))\n    \n    x = np.arange(1,len(train_acc_list)+1)\n    \n    ax[0].set_title('Accuracies')\n    ax[0].plot(x,train_acc_list, color = 'green', label = 'training')\n    ax[0].plot(x,valid_acc_list, color = 'blue', label = 'validation')\n    \n    \n    annot_max(x,np.array(valid_acc_list),ax[0])\n    ax[0].legend()\n\n    ax[1].set_title('Losses')\n    ax[1].plot(x,train_loss_list, color = 'green', label = 'training')\n    ax[1].plot(x,valid_loss_list, color = 'blue', label = 'validation')\n    \n    ax[1].legend()\n\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.009721Z","iopub.execute_input":"2022-04-05T09:59:52.01014Z","iopub.status.idle":"2022-04-05T09:59:52.023389Z","shell.execute_reply.started":"2022-04-05T09:59:52.010106Z","shell.execute_reply":"2022-04-05T09:59:52.022332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\nI used the model in https://www.kaggle.com/code/nicapotato/pytorch-cnn-kanada/notebook.","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, dropout = 0.40):\n        super(Net, self).__init__()\n        self.dropout = dropout\n        \n        # https://blog.algorithmia.com/convolutional-neural-nets-in-pytorch\n        #Our batch shape for input x is (1, 28, 28)\n        # (Batch, Number Channels, height, width).\n        #Input channels = 1, output channels = 18\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=5, stride=1, padding=2)\n        self.conv1_bn = nn.BatchNorm2d(num_features=64)\n        \n        self.conv1_1 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=1, padding=2)\n        self.conv1_1_bn = nn.BatchNorm2d(num_features=64)\n        self.pool1 = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_1 = nn.Dropout2d(p=self.dropout)\n        \n        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)\n        self.conv2_bn = nn.BatchNorm2d(num_features=128)\n        self.pool2 = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_2 = nn.Dropout2d(p=self.dropout)\n        \n        self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)\n        self.conv3_bn = nn.BatchNorm2d(num_features=256)\n        self.pool3 = nn.AvgPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_3 = nn.Dropout2d(p=self.dropout)\n        \n        #4608 input features, 256 output features (see sizing flow below)\n        self.fc1 = nn.Linear(256 * 3 * 3, 512) # Linear 1\n        self.d1_1 = nn.Dropout(p=self.dropout)\n        #64 input features, 10 output features for our 10 defined classes\n        self.fc2 = nn.Linear(in_features=512, out_features=256) # linear 2\n        self.d1_2 = nn.Dropout(p=self.dropout)\n        self.fc3 = nn.Linear(in_features=256, out_features=128) # linear 3\n        self.d1_3 = nn.Dropout(p=self.dropout)\n        self.out = nn.Linear(in_features=128, out_features=10) # linear 3\n        \n    def forward(self, x):\n        #Computes the activation of the first convolution\n        #Size changes from (1, 28, 28) to (18, 28, 28)\n        x = self.conv1(x)\n        x = self.conv1_bn(x)\n        x = F.relu(x)\n        x = self.conv1_1(x)\n        x = self.conv1_1_bn(x)\n        x = F.relu(x)       \n        \n        x = self.d2_1(x)\n        x = self.pool1(x) # Size changes from (18, 28, 28) to (18, 14, 14)\n        \n        # Second Conv       \n        x = self.conv2(x)\n        x = self.conv2_bn(x)\n        x = F.relu(x)\n        x = self.d2_2(x)\n        x = self.pool2(x) # Size changes from (18, 14, 14) to (18, 7, 7)\n        \n        # Third Conv       \n        x = self.conv3(x)\n        x = self.conv3_bn(x)\n        x = F.relu(x)\n        x = self.d2_3(x)\n        x = self.pool3(x) # Size changes from (18, 7, 7) to (18, 3, 3)\n        \n        #Reshape data to input to the input layer of the neural net\n        #Size changes from (18, 14, 14) to (1, 3528)\n        #Recall that the -1 infers this dimension from the other given dimension\n        x = x.view(-1, 256 * 3 * 3)\n\n        #Computes the activation of the first fully connected layer\n        #Size changes from (1, 4608) to (1, 64)\n        #Computes the second fully connected layer (activation applied later)\n        #Size changes from (1, 64) to (1, 10)\n        x = F.relu(self.fc1(x))\n        x = self.d1_1(x)\n        \n        x = F.relu(self.fc2(x))\n        x = self.d1_2(x)\n        \n        x = F.relu(self.fc3(x))\n        x = self.d1_3(x)\n        \n        x = self.out(x)\n        return F.log_softmax(x, dim=-1)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.025081Z","iopub.execute_input":"2022-04-05T09:59:52.025328Z","iopub.status.idle":"2022-04-05T09:59:52.04678Z","shell.execute_reply.started":"2022-04-05T09:59:52.025296Z","shell.execute_reply":"2022-04-05T09:59:52.046086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.049626Z","iopub.execute_input":"2022-04-05T09:59:52.049901Z","iopub.status.idle":"2022-04-05T09:59:52.058449Z","shell.execute_reply.started":"2022-04-05T09:59:52.049874Z","shell.execute_reply":"2022-04-05T09:59:52.057445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learning Rate Finder https://sgugger.github.io/how-do-you-find-a-good-learning-rate.html\n# https://www.kaggle.com/code/nicapotato/pytorch-cnn-kanada/notebook\ndef find_lr(trn_loader, init_value = 1e-8, final_value=10., beta = 0.98):\n    num = len(trn_loader)-1\n    mult = (final_value / init_value) ** (1/num)\n    lr = init_value\n    optimizer.param_groups[0]['lr'] = lr\n    avg_loss = 0.\n    best_loss = 0.\n    batch_num = 0\n    losses = []\n    log_lrs = []\n    for data in trn_loader:\n        batch_num += 1\n        #As before, get the loss for this mini-batch of inputs/outputs\n        inputs = data[0].to(device)\n        labels = data[1].to(device)\n        optimizer.zero_grad()\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        #Compute the smoothed loss\n        avg_loss = beta * avg_loss + (1-beta)*loss.item()\n        smoothed_loss = avg_loss / (1 - beta**batch_num)\n        #Stop if the loss is exploding\n        if batch_num > 1 and smoothed_loss > 4 * best_loss:\n            return log_lrs, losses\n        #Record the best loss\n        if smoothed_loss < best_loss or batch_num==1:\n            best_loss = smoothed_loss\n        #Store the values\n        losses.append(smoothed_loss)\n        log_lrs.append(math.log10(lr))\n        #Do the SGD step\n        loss.backward()\n        optimizer.step()\n        #Update the lr for the next step\n        lr *= mult\n        optimizer.param_groups[0]['lr'] = lr\n    return log_lrs, losses","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.060661Z","iopub.execute_input":"2022-04-05T09:59:52.061662Z","iopub.status.idle":"2022-04-05T09:59:52.072017Z","shell.execute_reply.started":"2022-04-05T09:59:52.061625Z","shell.execute_reply":"2022-04-05T09:59:52.071199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(42)\nnet = Net().to(device)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.073732Z","iopub.execute_input":"2022-04-05T09:59:52.074498Z","iopub.status.idle":"2022-04-05T09:59:52.102871Z","shell.execute_reply.started":"2022-04-05T09:59:52.074461Z","shell.execute_reply":"2022-04-05T09:59:52.102094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(net.parameters(), lr=1e-3)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.104103Z","iopub.execute_input":"2022-04-05T09:59:52.104819Z","iopub.status.idle":"2022-04-05T09:59:52.110665Z","shell.execute_reply.started":"2022-04-05T09:59:52.10478Z","shell.execute_reply":"2022-04-05T09:59:52.109936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logs,losses = find_lr(trn_loader = train_loader)\nplt.plot(logs[10:-5],losses[10:-5])","metadata":{"execution":{"iopub.status.busy":"2022-04-05T09:59:52.112183Z","iopub.execute_input":"2022-04-05T09:59:52.112787Z","iopub.status.idle":"2022-04-05T10:00:05.225444Z","shell.execute_reply.started":"2022-04-05T09:59:52.112746Z","shell.execute_reply":"2022-04-05T10:00:05.224788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary(net, input_size = (1, 28, 28), device = 'cuda')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:00:05.229304Z","iopub.execute_input":"2022-04-05T10:00:05.23122Z","iopub.status.idle":"2022-04-05T10:00:05.236417Z","shell.execute_reply.started":"2022-04-05T10:00:05.231181Z","shell.execute_reply":"2022-04-05T10:00:05.235487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://discuss.pytorch.org/t/how-to-do-exponential-learning-rate-decay-in-pytorch/63146\ntorch.manual_seed(42)\nnet = Net().to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(net.parameters(), lr=4e-3)\ndecayRate = 0.95\nlr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=decayRate)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:00:05.238433Z","iopub.execute_input":"2022-04-05T10:00:05.238962Z","iopub.status.idle":"2022-04-05T10:00:05.284927Z","shell.execute_reply.started":"2022-04-05T10:00:05.238926Z","shell.execute_reply":"2022-04-05T10:00:05.28431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_dict = train(train_loader,valid_loader,net,optimizer,criterion,40,lr_scheduler)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:00:05.288654Z","iopub.execute_input":"2022-04-05T10:00:05.290595Z","iopub.status.idle":"2022-04-05T10:03:51.391826Z","shell.execute_reply.started":"2022-04-05T10:00:05.290557Z","shell.execute_reply":"2022-04-05T10:03:51.39105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_dict","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:51.393158Z","iopub.execute_input":"2022-04-05T10:03:51.393547Z","iopub.status.idle":"2022-04-05T10:03:51.404411Z","shell.execute_reply.started":"2022-04-05T10:03:51.393511Z","shell.execute_reply":"2022-04-05T10:03:51.403507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_acc_loss(result_dict)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:51.40619Z","iopub.execute_input":"2022-04-05T10:03:51.406464Z","iopub.status.idle":"2022-04-05T10:03:51.751544Z","shell.execute_reply.started":"2022-04-05T10:03:51.406431Z","shell.execute_reply":"2022-04-05T10:03:51.750864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\n#test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:51.75555Z","iopub.execute_input":"2022-04-05T10:03:51.755767Z","iopub.status.idle":"2022-04-05T10:03:52.092188Z","shell.execute_reply.started":"2022-04-05T10:03:51.755742Z","shell.execute_reply":"2022-04-05T10:03:52.09148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids, test_set = test_data.id, test_data.drop(['id'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:52.093257Z","iopub.execute_input":"2022-04-05T10:03:52.093511Z","iopub.status.idle":"2022-04-05T10:03:52.106189Z","shell.execute_reply.started":"2022-04-05T10:03:52.093479Z","shell.execute_reply":"2022-04-05T10:03:52.105378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = torch.from_numpy(test_set.values)\ntest_set = test_set.view(test_set.shape[0],28,28).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:52.107684Z","iopub.execute_input":"2022-04-05T10:03:52.108318Z","iopub.status.idle":"2022-04-05T10:03:52.121405Z","shell.execute_reply.started":"2022-04-05T10:03:52.108271Z","shell.execute_reply":"2022-04-05T10:03:52.120816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:52.122495Z","iopub.execute_input":"2022-04-05T10:03:52.122763Z","iopub.status.idle":"2022-04-05T10:03:52.127753Z","shell.execute_reply.started":"2022-04-05T10:03:52.122729Z","shell.execute_reply":"2022-04-05T10:03:52.127064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set.max()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:52.129185Z","iopub.execute_input":"2022-04-05T10:03:52.129646Z","iopub.status.idle":"2022-04-05T10:03:52.13988Z","shell.execute_reply.started":"2022-04-05T10:03:52.129611Z","shell.execute_reply":"2022-04-05T10:03:52.139229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = test_set/255.","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:52.141298Z","iopub.execute_input":"2022-04-05T10:03:52.141533Z","iopub.status.idle":"2022-04-05T10:03:52.145274Z","shell.execute_reply.started":"2022-04-05T10:03:52.141502Z","shell.execute_reply":"2022-04-05T10:03:52.144506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions=[]\nwith torch.no_grad():\n    for i, data in enumerate(test_set):\n        images = data.to(device)\n        outputs = net(images.float().unsqueeze(0).unsqueeze(0))\n        \n        predicted = outputs.data.cpu().numpy().argmax()\n        #print(f\"{outputs.data} \\n ---> {predicted} \\n----------------\")\n        predictions.append([i,predicted.item()])","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:52.146431Z","iopub.execute_input":"2022-04-05T10:03:52.146973Z","iopub.status.idle":"2022-04-05T10:03:59.718932Z","shell.execute_reply.started":"2022-04-05T10:03:52.146933Z","shell.execute_reply":"2022-04-05T10:03:59.718204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:59.720253Z","iopub.execute_input":"2022-04-05T10:03:59.720564Z","iopub.status.idle":"2022-04-05T10:03:59.727105Z","shell.execute_reply.started":"2022-04-05T10:03:59.720527Z","shell.execute_reply":"2022-04-05T10:03:59.726279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions=pd.DataFrame(predictions)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-04-05T10:03:59.728559Z","iopub.execute_input":"2022-04-05T10:03:59.728836Z","iopub.status.idle":"2022-04-05T10:03:59.740501Z","shell.execute_reply.started":"2022-04-05T10:03:59.728802Z","shell.execute_reply":"2022-04-05T10:03:59.739742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.columns=['id','label']","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:59.741785Z","iopub.execute_input":"2022-04-05T10:03:59.7421Z","iopub.status.idle":"2022-04-05T10:03:59.748593Z","shell.execute_reply.started":"2022-04-05T10:03:59.742067Z","shell.execute_reply":"2022-04-05T10:03:59.747936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:59.749924Z","iopub.execute_input":"2022-04-05T10:03:59.750197Z","iopub.status.idle":"2022-04-05T10:03:59.766724Z","shell.execute_reply.started":"2022-04-05T10:03:59.750164Z","shell.execute_reply":"2022-04-05T10:03:59.766115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.__version__","metadata":{"execution":{"iopub.status.busy":"2022-04-05T10:03:59.768369Z","iopub.execute_input":"2022-04-05T10:03:59.768735Z","iopub.status.idle":"2022-04-05T10:03:59.773959Z","shell.execute_reply.started":"2022-04-05T10:03:59.768689Z","shell.execute_reply":"2022-04-05T10:03:59.773121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}