{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport json, sys, os, random\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.models as models\n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.395497Z","iopub.execute_input":"2022-10-30T11:59:33.395858Z","iopub.status.idle":"2022-10-30T11:59:33.406392Z","shell.execute_reply.started":"2022-10-30T11:59:33.395828Z","shell.execute_reply":"2022-10-30T11:59:33.405434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"61ba8bda-204d-45de-b20c-12c6a4220936","_cell_guid":"5b67ee9c-14ed-49f0-94c6-592c00f1efd8","execution":{"iopub.status.busy":"2022-10-30T11:59:33.410269Z","iopub.execute_input":"2022-10-30T11:59:33.410650Z","iopub.status.idle":"2022-10-30T11:59:33.419003Z","shell.execute_reply.started":"2022-10-30T11:59:33.410610Z","shell.execute_reply":"2022-10-30T11:59:33.417854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class KannadaMNIST(Dataset):\n    def __init__(self, images, labels=None, transform=None, mode='TRAIN'):\n        self.images = images\n        self.labels = labels\n        self.transform = transform\n        self.mode = mode\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, index):\n        \n        inputs = self.images.iloc[index,:]\n        inputs = np.array(inputs).astype(np.uint8).reshape(28, 28, 1)\n        inputs = self.transform(inputs)\n    \n        if self.mode == \"TEST\":\n            return inputs\n        else:\n            targets = self.labels.iloc[index]\n            return inputs, targets","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.420456Z","iopub.execute_input":"2022-10-30T11:59:33.421412Z","iopub.status.idle":"2022-10-30T11:59:33.429275Z","shell.execute_reply.started":"2022-10-30T11:59:33.421377Z","shell.execute_reply":"2022-10-30T11:59:33.428339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_prep(batch_size = 256, num_workers = 16, test_size = 0.4, shuffle = True):\n    file_url = '../input/Kannada-MNIST/train.csv'\n    dt = pd.read_csv(file_url)\n    \n    image_data = dt.drop('label', axis=1)\n    image_label = dt.label\n    \n    batch_size = batch_size\n    num_workers = num_workers\n    test_size = test_size\n    shuffle = shuffle\n    \n    trainX, validX, trainY, validY = train_test_split(image_data, image_label, test_size=test_size, random_state=42)\n    \n    trainX.reset_index(drop=True, inplace=True)\n    validX.reset_index(drop=True, inplace=True)\n    trainY.reset_index(drop=True, inplace=True)\n    validY.reset_index(drop=True, inplace=True)\n\n    train_transformer = transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.RandomAffine(degrees=10, translate=(0.1, 0.1)),\n        transforms.ToTensor()\n    ])\n    \n    valid_transformer = transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.ToTensor()\n        \n    ])\n\n    trainingDataset = KannadaMNIST(trainX, trainY, train_transformer)\n    \n    validationDataset = KannadaMNIST(validX, validY, valid_transformer)\n    \n    trainDataLoader = DataLoader(trainingDataset, \n                             batch_size=batch_size, \n                             num_workers=num_workers, shuffle=shuffle)\n    \n    validDataLoader = DataLoader(validationDataset, \n                                batch_size=batch_size, \n                                num_workers=num_workers, shuffle=shuffle,)\n    \n    return trainingDataset, validationDataset, trainDataLoader, validDataLoader\n","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.431161Z","iopub.execute_input":"2022-10-30T11:59:33.432008Z","iopub.status.idle":"2022-10-30T11:59:33.442933Z","shell.execute_reply.started":"2022-10-30T11:59:33.431971Z","shell.execute_reply":"2022-10-30T11:59:33.441921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class SimpleConvNet0(nn.Module):\n    \n#     def __init__(self):\n#         super(SimpleConvNet0, self).__init__()\n#         # Build Convolution Layer Section for our ConvNet\n#         self.conv2D_1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)\n#         self.batchNorm_1 = nn.BatchNorm2d(32)\n        \n#         self.conv2D_2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)\n#         self.batchNorm_2 = nn.BatchNorm2d(64)\n        \n#         self.conv2D_3 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)\n#         self.batchNorm_3 = nn.BatchNorm2d(128)\n        \n#         # next we are going to build the Fully Connected Layer or Dense Layer\n#         self.FC_1  = nn.Linear(in_features=1152, out_features=128)\n#         self.FC_BN_1 = nn.BatchNorm1d(num_features=128)\n        \n#         self.FC_2 = nn.Linear(in_features=128, out_features=10)\n        \n#         self.dropout_rate = 0.5\n    \n#     def forward(self, x):\n#         # apply all the conv layers we have just created before\n#         x = self.batchNorm_1(self.conv2D_1(x))      # dim batch_size x 32 x 256 x 256\n#         x = F.relu(F.max_pool2d(x, kernel_size=2))  # dim batch_size x 32 x 128 x 128\n        \n#         x = self.batchNorm_2(self.conv2D_2(x))      # dim batch_size x 64 x 128 x 128\n#         x = F.relu(F.max_pool2d(x, kernel_size=2))  # dim batch_size x 64 x 64 x 64\n        \n#         x = self.batchNorm_3(self.conv2D_3(x))      # dim batch_size x 128 x 64 x 64\n#         x = F.relu(F.max_pool2d(x, 2))              # dim batch_szie x 128 x 32 x 32\n\n#         # apply all the dense layer using output from conv layers\n#         x = x.view(-1, 1152) # change input dimension to 1D\n        \n#         x = self.FC_1(x)\n#         x = self.FC_BN_1(x)\n#         x = F.relu(x)\n        \n#         x = F.dropout(x, self.dropout_rate)\n#         x = self.FC_2(x)\n#         return x","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.444296Z","iopub.execute_input":"2022-10-30T11:59:33.445056Z","iopub.status.idle":"2022-10-30T11:59:33.455748Z","shell.execute_reply.started":"2022-10-30T11:59:33.445015Z","shell.execute_reply":"2022-10-30T11:59:33.454801Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleConvNet(nn.Module):\n    \n    def __init__(self):\n        super(SimpleConvNet, self).__init__()\n        # Build Convolution Layer Section for our ConvNet\n        #1st Block\n        self.conv2D_1 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_1 = nn.BatchNorm2d(64)\n        self.conv2D_2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_2 = nn.BatchNorm2d(64)\n        self.conv2D_3 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_3 = nn.BatchNorm2d(64)\n        #2nd Block\n        self.conv2D_4 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_4 = nn.BatchNorm2d(128)\n        self.conv2D_5 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_5 = nn.BatchNorm2d(128)\n        self.conv2D_6 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_6 = nn.BatchNorm2d(128)\n        #3rd Block\n        self.conv2D_7 = nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_7 = nn.BatchNorm2d(256)\n        self.conv2D_8 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)\n        self.batchNorm_8 = nn.BatchNorm2d(256)\n        \n        # next we are going to build the Fully Connected Layer or Dense Layer\n        self.FC_1  = nn.Linear(in_features=256*3*3, out_features=256)\n        self.FC_BN_1 = nn.BatchNorm1d(num_features=256)\n        self.FC_2 = nn.Linear(in_features=256, out_features=10)\n        \n        #define dropout rate\n        self.dropout_rate = 0.25\n    \n    def forward(self, x):\n        # apply all the conv layers we have just created before\n        # 1st Block\n        x = self.batchNorm_1(self.conv2D_1(x))\n        x = F.leaky_relu(x)\n        x = self.batchNorm_2(self.conv2D_2(x))\n        x = F.leaky_relu(x)\n        x = self.batchNorm_3(self.conv2D_3(x))\n        x = F.leaky_relu(x)\n        x = F.max_pool2d(x, kernel_size=2)     \n        x = F.dropout(x, self.dropout_rate)\n        # 2nd Block\n        x = self.batchNorm_4(self.conv2D_4(x))\n        x = F.leaky_relu(x)\n        x = self.batchNorm_5(self.conv2D_5(x))\n        x = F.leaky_relu(x)\n        x = self.batchNorm_6(self.conv2D_6(x))\n        x = F.leaky_relu(x)\n        x = F.max_pool2d(x, kernel_size=2)     \n        x = F.dropout(x, self.dropout_rate)\n        # 3rd Block\n        x = self.batchNorm_7(self.conv2D_7(x))\n        x = F.leaky_relu(x)\n        x = self.batchNorm_8(self.conv2D_8(x))\n        x = F.leaky_relu(x)\n        x = F.max_pool2d(x, kernel_size=2)     \n        x = F.dropout(x, self.dropout_rate)\n        #print(x.size())\n        x = x.view(-1, 256*3*3)\n        #Fully Connected Block\n        x = F.leaky_relu(self.FC_BN_1(self.FC_1(x)))\n        x = F.dropout(x, self.dropout_rate)\n        x = self.FC_2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.653265Z","iopub.execute_input":"2022-10-30T11:59:33.653544Z","iopub.status.idle":"2022-10-30T11:59:33.671983Z","shell.execute_reply.started":"2022-10-30T11:59:33.653519Z","shell.execute_reply":"2022-10-30T11:59:33.670938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trainingDataset, validationDataset, trainDataLoader, validDataLoader  = data_prep(batch_size = 64, \n#                                                                                   num_workers = 32, \n#                                                                                   test_size = 0.25, \n#                                                                                   shuffle = True)\n# inputs, targets = next(iter(trainDataLoader))\n# model = SimpleConvNet()\n# outputs = model(inputs)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.674557Z","iopub.execute_input":"2022-10-30T11:59:33.675098Z","iopub.status.idle":"2022-10-30T11:59:33.685006Z","shell.execute_reply.started":"2022-10-30T11:59:33.675063Z","shell.execute_reply":"2022-10-30T11:59:33.684094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n        device = torch.device(\"cuda\")\n\nmodel = SimpleConvNet()\nmodel.to(device)\n# model = models.vgg11()\n# model.features[0] = nn.Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n# model.classifier[-1] = nn.Linear(in_features=4096, out_features=10, bias=True)\n# model.to(device)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.688477Z","iopub.execute_input":"2022-10-30T11:59:33.688730Z","iopub.status.idle":"2022-10-30T11:59:33.733728Z","shell.execute_reply.started":"2022-10-30T11:59:33.688706Z","shell.execute_reply":"2022-10-30T11:59:33.732295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# def train_model(model, optimizer, loss_fn, trainLoader, validLoader, epochs=10, device=\"cpu\"):\n    \n#     best_accuracy = 0\n#     best_model = []\n    \n#     for epoch in range(epochs):\n        \n#         print(\"Epoch {} :\".format(epoch))\n        \n#         training_loss = 0.0\n#         validation_loss = 0.0\n#         model.train(True)\n#         for inputs, targets in tqdm(trainLoader):\n#             inputs, targets = inputs.to(device), targets.to(device)\n            \n#             optimizer.zero_grad()\n#             outputs = model(inputs)\n#             loss = loss_fn(outputs, targets)\n            \n#             loss.backward()\n#             optimizer.step()\n            \n#             training_loss += loss.data.item() * inputs.size(0)\n            \n#         training_loss /= len(trainLoader.dataset)\n\n        \n#         n_correct = 0\n#         n_examples = 0\n#         model.train(False)\n#         for inputs, targets in tqdm(validLoader):\n            \n#             inputs, targets = inputs.to(device), targets.to(device)\n            \n#             outputs = model(inputs)\n#             loss = loss_fn(outputs, targets)\n            \n#             validation_loss += loss.data.item() * inputs.size(0)\n            \n#             correct = torch.eq(\n#                 torch.max(F.softmax(outputs, dim=1), dim=1)[1], \n#                 targets \n#             ).view(-1)\n            \n#             n_correct += torch.sum(correct).item()\n#             n_examples += correct.shape[0]\n            \n#         validation_loss /= len(validLoader.dataset)\n        \n#         accuracy = n_correct/n_examples\n        \n#         if accuracy > best_accuracy:\n#             best_model = model\n#             best_accuracy = accuracy\n            \n#         print(\"Avg Train Loss : {:.3f}, \".format(training_loss), end='')\n#         print(\"Avg Val Loss : {:.3f}, \".format(validation_loss), end='')\n#         print(\"Val Acc : {:.3f}\".format(accuracy))\n#         print(\"=================================================\")\n        \n#     return best_model","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.736477Z","iopub.execute_input":"2022-10-30T11:59:33.737515Z","iopub.status.idle":"2022-10-30T11:59:33.753564Z","shell.execute_reply.started":"2022-10-30T11:59:33.737472Z","shell.execute_reply":"2022-10-30T11:59:33.752366Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.RMSprop(model.parameters(), lr=0.002)\nloss_func = torch.nn.CrossEntropyLoss()\n\ntrainingDataset, validationDataset, trainDataLoader, validDataLoader  = data_prep(batch_size = 64, \n                                                                                  num_workers = 32, \n                                                                                  test_size = 0.3, \n                                                                                  shuffle = True)\n# best_model = train_model(model, optimizer, loss_func, trainDataLoader, validDataLoader, epochs, device)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:33.757509Z","iopub.execute_input":"2022-10-30T11:59:33.758198Z","iopub.status.idle":"2022-10-30T11:59:36.924156Z","shell.execute_reply.started":"2022-10-30T11:59:33.758164Z","shell.execute_reply":"2022-10-30T11:59:36.923158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import notebook\n# optimizer = optim.Adam(model.parameters(), lr=0.001)\nepochs = 50\nbest_accuracy = 0\nbest_model = []\n\nfor epoch in range(epochs):\n    \n    if epoch % 10 == 0:\n        for g in optimizer.param_groups:\n            g['lr'] = g['lr'] * 0.1\n        \n    print(\"Epoch {} :\".format(epoch))\n\n    training_loss = 0.0\n    validation_loss = 0.0\n    model.train(True)\n    for i, (inputs, targets) in enumerate(tqdm(trainDataLoader)):\n\n        inputs, targets = inputs.to(device), targets.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = loss_func(outputs, targets)\n        loss.backward()\n        optimizer.step()\n\n        training_loss += loss.data.item() * inputs.size(0)\n\n    training_loss /= len(trainDataLoader.dataset)\n\n\n    n_correct = 0\n    n_examples = 0\n    model.train(False)\n    for i, (inputs, targets) in enumerate(tqdm(validDataLoader)):\n\n        inputs, targets = inputs.to(device), targets.to(device)\n\n        outputs = model(inputs)\n        loss = loss_func(outputs, targets)\n\n        validation_loss += loss.data.item() * inputs.size(0)\n\n        correct = torch.eq(\n            torch.max(F.softmax(outputs, dim=1), dim=1)[1], \n            targets \n        ).view(-1)\n\n        n_correct += torch.sum(correct).item()\n        n_examples += correct.shape[0]\n\n    validation_loss /= len(validDataLoader.dataset)\n\n    accuracy = n_correct/n_examples\n\n    if accuracy > best_accuracy:\n        best_model = model\n        best_accuracy = accuracy\n\n    print(\"Avg Train Loss : {:.3f}, \".format(training_loss), end='')\n    print(\"Avg Val Loss : {:.3f}, \".format(validation_loss), end='')\n    print(\"Val Acc : {:.3f}\".format(accuracy))\n    print(\"=================================================\")","metadata":{"execution":{"iopub.status.busy":"2022-10-30T11:59:36.925779Z","iopub.execute_input":"2022-10-30T11:59:36.926180Z","iopub.status.idle":"2022-10-30T12:25:46.233084Z","shell.execute_reply.started":"2022-10-30T11:59:36.926144Z","shell.execute_reply":"2022-10-30T12:25:46.230733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(model)\n# x = model.batchNorm_1(model.conv2D_1(inputs))\n# x = model.batchNorm_2(model.conv2D_2(x))\n# x = model.batchNorm_3(model.conv2D_3(x))\n# x = model.batchNorm_4(model.conv2D_4(x))\n# print(x.size())\n# x = x.view(-1, 128*28*28)\n# print(x.size())\n# x = model.FC_BN_1(model.FC_1(x))\n","metadata":{"execution":{"iopub.status.busy":"2022-10-30T12:25:46.236627Z","iopub.execute_input":"2022-10-30T12:25:46.240026Z","iopub.status.idle":"2022-10-30T12:25:46.249101Z","shell.execute_reply.started":"2022-10-30T12:25:46.239972Z","shell.execute_reply":"2022-10-30T12:25:46.247925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TESTING MODEL\n\ntest_file = '../input/Kannada-MNIST/test.csv'\nnum_workers = 4\nshuffle = False\n\n#load testing data file\ndt = pd.read_csv(test_file)\nimage_data = dt.drop('id', axis=1)\nimage_label = None\nimage_id = dt.id.tolist()\nbatch_size = 32\n\n#create test transformer\ntest_transformer = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor()\n])\n\n#create test Dataset\ntestingDataset = KannadaMNIST(image_data, image_label, test_transformer, mode='TEST')\n\n#create test Dataloader\ntestDataloader = DataLoader(testingDataset, \n                            batch_size=batch_size, \n                            num_workers=num_workers, shuffle=shuffle)\n\n#load model state dictionary\nif torch.cuda.is_available():\n    device = torch.device(\"cuda\")\nelse:\n    sys.exit(\"No CUDA Supported device detected \")\n\nmodel = best_model\n# call model.eval() before testing process begin.\nmodel.eval()\npreds = []\n# begin model testing process\nprint('<',end='')\nfor inputs in testDataloader:\n    print('=',end='')\n    inputs = inputs.to(device)\n    outputs = model(inputs)\n    preds += (torch.max(F.softmax(outputs, dim=1), dim=1)[1].cpu().numpy().tolist())\n    \nprint('> Done')\n\noutfile = 'submission.csv'\nresults = {'id':image_id, 'label':preds}\ndf = pd.DataFrame(data=results)\ndf.to_csv(outfile, index=False)\nprint('Finished creating subsmission file {}'.format(outfile))","metadata":{"execution":{"iopub.status.busy":"2022-10-30T12:34:25.390388Z","iopub.execute_input":"2022-10-30T12:34:25.390744Z","iopub.status.idle":"2022-10-30T12:34:27.621143Z","shell.execute_reply.started":"2022-10-30T12:34:25.390711Z","shell.execute_reply":"2022-10-30T12:34:27.619707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}