{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np \nimport pandas as pd \nimport os\nfrom sklearn.metrics import accuracy_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom torchvision.transforms import ToTensor\nfrom torchvision.models import resnet as model_res\nimport warnings\nwarnings.filterwarnings('ignore')\nimport random\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm import tqdm\n# Any results you write to the current directory are saved as output.","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-10-08T10:40:05.554996Z","iopub.execute_input":"2022-10-08T10:40:05.555705Z","iopub.status.idle":"2022-10-08T10:40:05.563214Z","shell.execute_reply.started":"2022-10-08T10:40:05.555668Z","shell.execute_reply":"2022-10-08T10:40:05.562135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(42)\nrandom.seed(42)\nnp.random.seed(42)\ntorch.backends.cudnn.enabled = False","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:40:06.240350Z","iopub.execute_input":"2022-10-08T10:40:06.241037Z","iopub.status.idle":"2022-10-08T10:40:06.246397Z","shell.execute_reply.started":"2022-10-08T10:40:06.240998Z","shell.execute_reply":"2022-10-08T10:40:06.245354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_classes = 10\nN_EPOCHS = 20\nBATCH_SIZE = 256\nIMG_DIM = 28\nrandom_seed = 42\ntorch.manual_seed(random_seed)\ntorch.backends.cudnn.enabled = False\n# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-10-08T10:42:15.762705Z","iopub.execute_input":"2022-10-08T10:42:15.763717Z","iopub.status.idle":"2022-10-08T10:42:15.770977Z","shell.execute_reply.started":"2022-10-08T10:42:15.763677Z","shell.execute_reply":"2022-10-08T10:42:15.769999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load in data to train/test with\ndir_data = '/kaggle/input/Kannada-MNIST'\ndir_train = dir_data +'/train.csv'\ndir_test = dir_data + '/test.csv'\nalternate_test = dir_data + '/Dig-MNIST.csv'\n\ndf_train = pd.read_csv(dir_train)\ndf_test = pd.read_csv(dir_test)\ndf_alternate_test = pd.read_csv(alternate_test)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:16.326942Z","iopub.execute_input":"2022-10-08T10:42:16.327320Z","iopub.status.idle":"2022-10-08T10:42:19.820862Z","shell.execute_reply.started":"2022-10-08T10:42:16.327289Z","shell.execute_reply":"2022-10-08T10:42:19.819596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop training label from training data\ntarget = df_train['label']\ndf_train.drop('label', axis=1, inplace=True)\n\n# drop id column as uselss\ndf_test.drop('id', axis=1, inplace=True)\n\n# drop label from alternate test data\nalternate_test_label = df_alternate_test['label']\ndf_alternate_test.drop('label', axis=1, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:19.823026Z","iopub.execute_input":"2022-10-08T10:42:19.823751Z","iopub.status.idle":"2022-10-08T10:42:19.996455Z","shell.execute_reply.started":"2022-10-08T10:42:19.823698Z","shell.execute_reply":"2022-10-08T10:42:19.995256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random train and test split on the training data.\n# alternate_test can be used as a holdout test to evaluate the model.\nX_train, X_test, y_train, y_test = train_test_split(df_train, target, stratify=target, random_state=42, test_size=0.1)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:19.998043Z","iopub.execute_input":"2022-10-08T10:42:19.998782Z","iopub.status.idle":"2022-10-08T10:42:20.454419Z","shell.execute_reply.started":"2022-10-08T10:42:19.998739Z","shell.execute_reply":"2022-10-08T10:42:20.453452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=10, ncols=10, figsize=(10,10))\n\n# I know these for loops look weird, but this way num_i is only computed once for each class\nfor i in range(10): # Column by column\n    num_i = X_train[y_train == i]\n    ax[0][i].set_title(i)\n    for j in range(10): # Row by row\n        ax[j][i].axis('off')\n        ax[j][i].imshow(num_i.iloc[j, :].to_numpy().astype(np.uint8).reshape(28, 28), cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:20.456966Z","iopub.execute_input":"2022-10-08T10:42:20.457354Z","iopub.status.idle":"2022-10-08T10:42:23.869343Z","shell.execute_reply.started":"2022-10-08T10:42:20.457316Z","shell.execute_reply":"2022-10-08T10:42:23.868252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class KannadaDataClass(Dataset):\n    def __init__(self, images, labels, transform, classes):\n        \"\"\"\n        instantiate KannadaData class with class artifacts\n        \"\"\"\n        self.X = images\n        self.y = labels\n        self.tranform = transform\n        self.classes = classes\n        \n    def __len__(self):\n        \"\"\"\n        mandatory function\n        \"\"\"\n        return len(self.X)\n    \n    def __getitem__(self, idx=None):\n        \"\"\"\n        will return label in softmax format [0,0,0,1,0,...]\n        reshape image data to (28,28,1)\n        return both image and label.\n        \"\"\"\n        img = np.array(self.X.iloc[idx,:], dtype='uint8').reshape((IMG_DIM,IMG_DIM,1))\n        if self.y is not None:\n            y = np.zeros(self.classes, dtype='float32')\n            y[self.y.iloc[idx]] = 1\n            return img, y\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.870910Z","iopub.execute_input":"2022-10-08T10:42:23.871371Z","iopub.status.idle":"2022-10-08T10:42:23.880678Z","shell.execute_reply.started":"2022-10-08T10:42:23.871336Z","shell.execute_reply":"2022-10-08T10:42:23.879198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put some augmentation on training data using pytorch torchvision.transforms\n\ntrain_transform = transforms.Compose([\n    transforms.Resize([IMG_DIM,IMG_DIM]),\n    #transforms.ToPILImage(),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1)),\n    transforms.ToTensor()\n])\n\n# test data is not augmented, kept as it is.\ntest_transform = transforms.Compose([\n    transforms.Resize([IMG_DIM,IMG_DIM]),\n    #transforms.ToPILImage(),\n    transforms.ToTensor()\n])","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.882281Z","iopub.execute_input":"2022-10-08T10:42:23.882725Z","iopub.status.idle":"2022-10-08T10:42:23.892809Z","shell.execute_reply.started":"2022-10-08T10:42:23.882658Z","shell.execute_reply":"2022-10-08T10:42:23.891820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create torch datasets for all datasets for easier path to training.\ntrain_dataset = KannadaDataClass(images=X_train, labels=y_train, transform=train_transform, classes=10)\ntest_dataset = KannadaDataClass(images=X_test, labels=y_test, transform=test_transform, classes=10)\nalternate_test_dataset = KannadaDataClass(images=df_alternate_test, labels=alternate_test_label, transform=test_transform, classes=10)\nfinal_result_test_dataset = KannadaDataClass(images=df_test, labels=None, transform=test_transform, classes=10)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.893869Z","iopub.execute_input":"2022-10-08T10:42:23.894124Z","iopub.status.idle":"2022-10-08T10:42:23.902791Z","shell.execute_reply.started":"2022-10-08T10:42:23.894101Z","shell.execute_reply":"2022-10-08T10:42:23.901895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Defining the data generators for producing batches of data\n# these dataLoaders are useful for training and evaluation due to how they return data\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False)\nalternate_test_loader = DataLoader(dataset=alternate_test_dataset, batch_size=BATCH_SIZE, shuffle=False)\nfinal_result_loader = DataLoader(dataset=final_result_test_dataset, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.903957Z","iopub.execute_input":"2022-10-08T10:42:23.904320Z","iopub.status.idle":"2022-10-08T10:42:23.912432Z","shell.execute_reply.started":"2022-10-08T10:42:23.904286Z","shell.execute_reply":"2022-10-08T10:42:23.911536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        \"\"\"\n        initialize net class attributes \n            including conv layers,\n            fully connected layers/dense layers\n            dropout\n        \"\"\"\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, 3, 1)\n        self.conv2 = nn.Conv2d(32, 64, 3, 1)\n        self.conv3 = nn.Conv2d(64, 128, 3, 1)\n        self.fc1 = nn.Linear(3200, 256)\n        self.fc2 = nn.Linear(256, 10)\n        self.dropout = nn.Dropout(0.25)\n\n    def forward(self, x):\n        \"\"\"\n        define CNN network architecture\n        Quite a general network following CNN development principles\n            of using relu, max pooling, dropout etc.\n        \"\"\"\n        x = self.conv1(x)\n        x = F.relu(x)\n        x = self.conv2(x)\n        x = F.max_pool2d(x, 2)\n        x = self.dropout(x)\n        x = F.relu(x)\n        x = self.conv3(x)\n        x = F.max_pool2d(x, 2)\n        x = self.dropout(x)\n        x = torch.flatten(x, 1)\n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        output = F.log_softmax(x, dim=1)\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.913802Z","iopub.execute_input":"2022-10-08T10:42:23.914369Z","iopub.status.idle":"2022-10-08T10:42:23.925470Z","shell.execute_reply.started":"2022-10-08T10:42:23.914335Z","shell.execute_reply":"2022-10-08T10:42:23.924594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Net()\nmodel = model.to(device)\n\noptimizer = optim.Adam(model.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.928751Z","iopub.execute_input":"2022-10-08T10:42:23.929162Z","iopub.status.idle":"2022-10-08T10:42:23.944933Z","shell.execute_reply.started":"2022-10-08T10:42:23.929137Z","shell.execute_reply":"2022-10-08T10:42:23.944076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(input, target, size_average=True):\n    \"\"\"\n    Categorical cross-entropy with logits input and one-hot target\n    \"\"\"\n    l = -(target * torch.log(F.softmax(input, dim=1) + 1e-10)).sum(1)\n    if size_average:\n        l = l.mean()\n    else:\n        l = l.sum()\n    return l","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.946293Z","iopub.execute_input":"2022-10-08T10:42:23.946710Z","iopub.status.idle":"2022-10-08T10:42:23.952808Z","shell.execute_reply.started":"2022-10-08T10:42:23.946676Z","shell.execute_reply":"2022-10-08T10:42:23.951360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(epoch, model, train_loader, device, optimizer, train_loss):\n    \"\"\"\n    function to train the model using gradient descent and back propogation\n    \"\"\"\n    model.train()\n    # function used to train the model\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data = data.to(device)\n        target = target.to(device)\n        # start of each batch zero our gradient\n        optimizer.zero_grad()\n        data = data.permute(0, 3, 1, 2).float()\n        output = model(data)\n        loss = criterion(output, target)\n        # calculate our loss for each batch\n        loss.backward()\n        # back propogate the error/loss through the network\n        optimizer.step()\n        \n        if (batch_idx + 1) % 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLR: {:.6f}\\tLoss: {:.6f}'.format(\n                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n                100. * (batch_idx + 1) / len(train_loader),\n                optimizer.state_dict()['param_groups'][0]['lr'],\n                loss.data))\n    train_loss.append(loss.data.cpu().numpy())\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.954104Z","iopub.execute_input":"2022-10-08T10:42:23.955046Z","iopub.status.idle":"2022-10-08T10:42:23.965799Z","shell.execute_reply.started":"2022-10-08T10:42:23.955017Z","shell.execute_reply":"2022-10-08T10:42:23.964736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, test_loader, alternate_test_loader, device, test_loss, test_accuracy, alternate_test_loss, alternate_test_accuracy):\n    \"\"\"\n    function to test model on test loss data\n    and on the alternate test data\n\n    \"\"\"\n    model.eval()\n    loss = 0\n    correct = 0\n    with torch.no_grad():\n        for data, target in test_loader:\n            data = data.to(device)\n            target = target.to(device)\n\n            data = data.permute(0, 3, 1, 2).float()\n            output = model(data)\n\n            loss += criterion(output, target, size_average=False).data\n\n            pred = output.data.max(1, keepdim=True)[1]\n            correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()\n    \n    loss /= len(test_loader.dataset)\n    accuracy = correct / len(test_loader.dataset)\n    \n    test_loss.append(loss.cpu().numpy())\n    test_accuracy.append(accuracy)\n\n    print('test loss: {:.4f}, test accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(test_loader.dataset),\n        100. * accuracy))     \n\n    correct = 0\n    loss = 0\n    with torch.no_grad():\n        for data, target in alternate_test_loader:\n            data = data.to(device)\n            target = target.to(device)\n            # permutation required to change order of dimensions in data variable\n            # the input channel should be in the second dimension in pytorch\n            data = data.permute(0, 3, 1, 2).float()\n            output = model(data)\n\n            loss += criterion(output, target, size_average=False).data\n\n            pred = output.data.max(1, keepdim=True)[1]\n            correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()\n    \n    loss /= len(alternate_test_loader.dataset)\n    accuracy = correct / len(alternate_test_loader.dataset)\n    \n    alternate_test_loss.append(loss.cpu().numpy())\n    alternate_test_accuracy.append(accuracy)\n    \n    print('alternate test loss: {:.4f}, alternate test accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(alternate_test_loader.dataset),\n        100. * accuracy))     \n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.967610Z","iopub.execute_input":"2022-10-08T10:42:23.968117Z","iopub.status.idle":"2022-10-08T10:42:23.981607Z","shell.execute_reply.started":"2022-10-08T10:42:23.968081Z","shell.execute_reply":"2022-10-08T10:42:23.980513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loss = []\ntest_loss = []\ntest_accuracy = []\nalternate_test_loss = []\nalternate_test_accuracy = []","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:23.983449Z","iopub.execute_input":"2022-10-08T10:42:23.983822Z","iopub.status.idle":"2022-10-08T10:42:23.993108Z","shell.execute_reply.started":"2022-10-08T10:42:23.983788Z","shell.execute_reply":"2022-10-08T10:42:23.992243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run training and evaluation\nfor epoch in range(N_EPOCHS):\n    print('Epoch: '+ str(epoch))\n    train(epoch,model,train_loader, device, optimizer, train_loss)\n    evaluate(model,test_loader, alternate_test_loader, device, test_loss, test_accuracy, alternate_test_loss, alternate_test_accuracy)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:42:24.098613Z","iopub.execute_input":"2022-10-08T10:42:24.098972Z","iopub.status.idle":"2022-10-08T10:50:05.175907Z","shell.execute_reply.started":"2022-10-08T10:42:24.098940Z","shell.execute_reply":"2022-10-08T10:50:05.174728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.eval()\n# predictions = []\n\n# for data in tqdm(dig_loader):\n#     data = data[0].to(device)\n#     data = data.permute(0, 3, 1, 2).float()\n#     output = model(data).max(dim=1)[1] # argmax\n#     predictions += list(output.data.cpu().numpy())\n\n# accuracy_score(dig_label, predictions)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:50:15.268643Z","iopub.execute_input":"2022-10-08T10:50:15.269075Z","iopub.status.idle":"2022-10-08T10:50:15.273740Z","shell.execute_reply.started":"2022-10-08T10:50:15.269040Z","shell.execute_reply":"2022-10-08T10:50:15.272812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alternate_test_accuracy","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:50:16.623148Z","iopub.execute_input":"2022-10-08T10:50:16.623841Z","iopub.status.idle":"2022-10-08T10:50:16.630490Z","shell.execute_reply.started":"2022-10-08T10:50:16.623803Z","shell.execute_reply":"2022-10-08T10:50:16.629365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(test_accuracy)\nplt.plot(alternate_test_accuracy)\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['test', 'dig'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:50:18.127078Z","iopub.execute_input":"2022-10-08T10:50:18.128512Z","iopub.status.idle":"2022-10-08T10:50:18.334018Z","shell.execute_reply.started":"2022-10-08T10:50:18.128464Z","shell.execute_reply":"2022-10-08T10:50:18.333091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict","metadata":{}},{"cell_type":"code","source":"model.eval()\npredictions = []\n\nfor data in tqdm(final_result_loader):\n    data = data.to(device)\n    data = data.permute(0, 3, 1, 2).float()\n    output = model(data).max(dim=1)[1] # argmax\n    predictions += list(output.data.cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:50:22.574472Z","iopub.execute_input":"2022-10-08T10:50:22.574933Z","iopub.status.idle":"2022-10-08T10:50:23.753539Z","shell.execute_reply.started":"2022-10-08T10:50:22.574891Z","shell.execute_reply":"2022-10-08T10:50:23.752526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:50:24.682419Z","iopub.execute_input":"2022-10-08T10:50:24.682780Z","iopub.status.idle":"2022-10-08T10:50:24.703738Z","shell.execute_reply.started":"2022-10-08T10:50:24.682749Z","shell.execute_reply":"2022-10-08T10:50:24.702684Z"},"trusted":true},"execution_count":null,"outputs":[]}]}