{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Convolutional Digit Classifier Kanada - Pytorch\n_October 1st 2019_\n\n**Kernels:**\n1. [Dense Net](https://www.kaggle.com/nicapotato/dense-digit-classifier-kanada-simple-cpu-pytorch)\n2. [Convolutional Neural Net](https://www.kaggle.com/nicapotato/pytorch-cnn-kanada)\n\n**Aim:** <br>\nBuild upon my simple dense net by introducing convolutional layers and well as some image pre-processing.\n\n**Additions:** <br>\n1. Convolutional, BatchNorm2d Layers\n2. Image Pre-Processing\n3. One Cycle Learning\n4. Additional Hold-Out set (Different Scanning Process)"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# Any results you write to the current directory are saved as output.\n\nimport torch\nimport torchvision\nfrom torchvision import transforms, datasets\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\nimport math\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data\ntrain=pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest=pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')\nsubmission_set = pd.read_csv(\"../input/Kannada-MNIST/test.csv\").iloc[:,1:]\n\ntrain_data=train.drop('label',axis=1)\ntrain_targets=train['label']\n\ntest_images=test.drop('label',axis=1)\ntest_labels=test['label']\n\n# Train Test Split\ntrain_images, val_images, train_labels, val_labels = train_test_split(train_data, \n                                                                     train_targets, \n                                                                     test_size=0.2)\n\n# Reset Index\ntrain_images.reset_index(drop=True, inplace=True)\ntrain_labels.reset_index(drop=True, inplace=True)\n\nval_images.reset_index(drop=True, inplace=True)\nval_labels.reset_index(drop=True, inplace=True)\n\ntest_images.reset_index(drop=True, inplace=True)\ntest_labels.reset_index(drop=True, inplace=True)\n\nprint(\"Train Set\")\nprint(train_images.shape)\nprint(train_labels.shape)\n\nprint(\"Validation Set\")\nprint(val_images.shape)\nprint(val_labels.shape)\n\nprint(\"Validation 2\")\nprint(test_images.shape)\nprint(test_labels.shape)\n\nprint(\"Submission\")\nprint(submission_set.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Look at image means\")\nprint(train_images.mean(axis = 1).mean())\nprint(val_images.mean(axis = 1).mean())\nprint(test_images.mean(axis = 1).mean())\nprint(submission_set.mean(axis = 1).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Distribution\")\nprint(train_labels.value_counts(normalize = True))\n\nprint(\"\\nSubmission Distribution\")\nprint(test_labels.value_counts(normalize = True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMGSIZE = 28\n\n# Transformations for the train\ntrain_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.RandomCrop(IMGSIZE),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1)),\n    transforms.ToTensor(), # divides by 255\n  #  transforms.Normalize((0.5,), (0.5,))\n]))\n\n# Transformations for the validation & test sets\nval_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.ToTensor(), # divides by 255\n   # transforms.Normalize((0.1307,), (0.3081,))\n]))\n\nclass KannadaDataSet(torch.utils.data.Dataset):\n    def __init__(self, images, labels,transforms = None):\n        self.X = images\n        self.y = labels\n        self.transforms = transforms\n         \n    def __len__(self):\n        return (len(self.X))\n    \n    def __getitem__(self, i):\n        data = self.X.iloc[i,:]\n        data = np.array(data).astype(np.uint8).reshape(IMGSIZE,IMGSIZE,1)\n        \n        if self.transforms:\n            data = self.transforms(data)\n            \n        if self.y is not None:\n            return (data, self.y[i])\n        else:\n            return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\ntrain_data = KannadaDataSet(train_images, train_labels, train_trans)\nval_data = KannadaDataSet(val_images, val_labels, val_trans)\ntest_data = KannadaDataSet(test_images, test_labels, val_trans)\nsubmission_data = KannadaDataSet(submission_set, None, val_trans)\n\n\ntrain_loader = torch.utils.data.DataLoader(train_data, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\nval_loader = torch.utils.data.DataLoader(val_data, \n                                           batch_size=batch_size, \n                                           shuffle=False)\n\ntest_loader = torch.utils.data.DataLoader(test_data,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_data,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, dropout = 0.40):\n        super(Net, self).__init__()\n        self.dropout = dropout\n        \n        # https://blog.algorithmia.com/convolutional-neural-nets-in-pytorch\n        #Our batch shape for input x is (1, 28, 28)\n        # (Batch, Number Channels, height, width).\n        #Input channels = 1, output channels = 18\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=5, stride=1, padding=2)\n        self.conv1_bn = nn.BatchNorm2d(num_features=64)\n        \n        self.conv1_1 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=1, padding=2)\n        self.conv1_1_bn = nn.BatchNorm2d(num_features=64)\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_1 = nn.Dropout2d(p=self.dropout)\n        \n        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)\n        self.conv2_bn = nn.BatchNorm2d(num_features=128)\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_2 = nn.Dropout2d(p=self.dropout)\n        \n        self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)\n        self.conv3_bn = nn.BatchNorm2d(num_features=256)\n        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_3 = nn.Dropout2d(p=self.dropout)\n        \n        #4608 input features, 256 output features (see sizing flow below)\n        self.fc1 = nn.Linear(256 * 3 * 3, 512) # Linear 1\n        self.d1_1 = nn.Dropout(p=self.dropout)\n        #64 input features, 10 output features for our 10 defined classes\n        self.fc2 = nn.Linear(in_features=512, out_features=256) # linear 2\n        self.d1_2 = nn.Dropout(p=self.dropout)\n        self.fc3 = nn.Linear(in_features=256, out_features=128) # linear 3\n        self.d1_3 = nn.Dropout(p=self.dropout)\n        self.out = nn.Linear(in_features=128, out_features=10) # linear 3\n        \n    def forward(self, x):\n        #Computes the activation of the first convolution\n        #Size changes from (1, 28, 28) to (18, 28, 28)\n        x = self.conv1(x)\n        x = self.conv1_bn(x)\n        x = F.relu(x)\n        x = self.conv1_1(x)\n        x = self.conv1_1_bn(x)\n        x = F.relu(x)       \n        \n        x = self.d2_1(x)\n        x = self.pool1(x) # Size changes from (18, 28, 28) to (18, 14, 14)\n        \n        # Second Conv       \n        x = self.conv2(x)\n        x = self.conv2_bn(x)\n        x = F.relu(x)\n        x = self.d2_2(x)\n        x = self.pool2(x) # Size changes from (18, 14, 14) to (18, 7, 7)\n        \n        # Third Conv       \n        x = self.conv3(x)\n        x = self.conv3_bn(x)\n        x = F.relu(x)\n        x = self.d2_3(x)\n        x = self.pool3(x) # Size changes from (18, 7, 7) to (18, 3, 3)\n        \n        #Reshape data to input to the input layer of the neural net\n        #Size changes from (18, 14, 14) to (1, 3528)\n        #Recall that the -1 infers this dimension from the other given dimension\n        x = x.view(-1, 256 * 3 * 3)\n\n        #Computes the activation of the first fully connected layer\n        #Size changes from (1, 4608) to (1, 64)\n        #Computes the second fully connected layer (activation applied later)\n        #Size changes from (1, 64) to (1, 10)\n        x = F.relu(self.fc1(x))\n        x = self.d1_1(x)\n        \n        x = F.relu(self.fc2(x))\n        x = self.d1_2(x)\n        \n        x = F.relu(self.fc3(x))\n        x = self.d1_3(x)\n        \n        x = self.out(x)\n        return F.log_softmax(x, dim=-1)\n\n\nnet = Net().to(device)\nnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def outputSize(in_size, kernel_size, stride, padding):\n    output = int((in_size - kernel_size + 2 * (padding)) / stride) + 1\n    return(output)\n# outputSize(64, 5, 1, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learning Rate Finder https://sgugger.github.io/how-do-you-find-a-good-learning-rate.html\ndef find_lr(trn_loader, init_value = 1e-8, final_value=10., beta = 0.98):\n    num = len(trn_loader)-1\n    mult = (final_value / init_value) ** (1/num)\n    lr = init_value\n    optimizer.param_groups[0]['lr'] = lr\n    avg_loss = 0.\n    best_loss = 0.\n    batch_num = 0\n    losses = []\n    log_lrs = []\n    for data in trn_loader:\n        batch_num += 1\n        #As before, get the loss for this mini-batch of inputs/outputs\n        inputs = data[0].to(device)\n        labels = data[1].to(device)\n        optimizer.zero_grad()\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        #Compute the smoothed loss\n        avg_loss = beta * avg_loss + (1-beta)*loss.item()\n        smoothed_loss = avg_loss / (1 - beta**batch_num)\n        #Stop if the loss is exploding\n        if batch_num > 1 and smoothed_loss > 4 * best_loss:\n            return log_lrs, losses\n        #Record the best loss\n        if smoothed_loss < best_loss or batch_num==1:\n            best_loss = smoothed_loss\n        #Store the values\n        losses.append(smoothed_loss)\n        log_lrs.append(math.log10(lr))\n        #Do the SGD step\n        loss.backward()\n        optimizer.step()\n        #Update the lr for the next step\n        lr *= mult\n        optimizer.param_groups[0]['lr'] = lr\n    return log_lrs, losses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = Net().to(device)\n\n# Loss Function\ncriterion = nn.CrossEntropyLoss()\n# criterion = F.nll_loss\n\n# Gradient Descent\n# optimizer = optim.SGD(net.parameters(),lr=1e-1)\noptimizer = optim.Adam(net.parameters(), lr=1e-1)\n\nlogs,losses = find_lr(trn_loader = train_loader)\nplt.plot(logs[10:-5],losses[10:-5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = Net().to(device)\n\nEPOCHS = 30\nnn_output = []\n\n# optimizer = optim.SGD(net.parameters(),lr=1e-2)\noptimizer = optim.Adam(net.parameters(), lr=4e-3)\ncriterion = nn.CrossEntropyLoss()\n# criterion = F.nll_loss\n\ndef get_num_correct(preds, labels):\n    return preds.argmax(dim=1).eq(labels).sum().item()\n\nfor epoch in range(EPOCHS):\n    epoch_loss = 0\n    epoch_correct = 0\n    net.train()\n    \n    for data in train_loader:\n        # `data` is a batch of data\n        # Before using transforms, I used .unsqueeze(1) to enter a empty number channel array (Batch, Number Channels, height, width).\n        X = data[0].to(device) # X is the batch of features\n        # Unsqueeze adds a placeholder dimension for the color channel - (8, 28, 28) to (8, 1, 28, 28)\n        y = data[1].to(device) # y is the batch of targets.\n        \n        net.zero_grad()  # sets gradients to 0 before loss calc. You will do this likely every step.\n        output = net(X)  # pass in the reshaped batch (recall they are 28x28 atm)\n        tloss = criterion(output, y)  # calc and grab the loss value\n        tloss.backward()  # apply this loss backwards thru the network's parameters\n        optimizer.step()  # attempt to optimize weights to account for loss/gradients \n        \n        epoch_loss += tloss.item()\n        epoch_correct += get_num_correct(output, y)\n    \n    # Evaluation with the validation set\n    net.eval() # eval mode\n    val_loss = 0\n    val_correct = 0\n    test_loss = 0\n    test_correct = 0\n    \n    with torch.no_grad():\n        # First Validation Set\n        for data in val_loader:\n            X = data[0].to(device)\n            y = data[1].to(device)\n            \n            preds = net(X) # get predictions\n            vloss = criterion(preds, y) # calculate the loss\n            \n            val_correct += get_num_correct(preds, y)\n            val_loss += vloss.item()\n        \n        # Second Validation Set..\n        for data in test_loader:\n            X = data[0].to(device)\n            y = data[1].to(device)\n            \n            preds = net(X) # get predictions\n            tstloss = criterion(preds, y) # calculate the loss\n            \n            test_correct += get_num_correct(preds, y)\n            test_loss += tstloss.item()\n    \n    tmp_nn_output = [epoch + 1,EPOCHS,\n                     epoch_loss/len(train_loader.dataset),epoch_correct/len(train_loader.dataset)*100,\n                     val_loss/len(val_loader.dataset), val_correct/len(val_loader.dataset)*100,\n                     test_loss/len(test_loader.dataset), test_correct/len(test_loader.dataset)*100\n                    ]\n    nn_output.append(tmp_nn_output)\n    \n    # Print the loss and accuracy for the validation set\n    print('Epoch [{}/{}] train loss: {:.6f} acc: {:.3f} - valid loss: {:.6f} acc: {:.3f} - Test loss: {:.6f} acc: {:.3f}'\n        .format(*tmp_nn_output))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_results = pd.DataFrame(nn_output,\n    columns = ['epoch','total_epochs','train_loss','train_acc','valid_loss','valid_acc','test_loss','test_acc']\n                         )\ndisplay(pd_results)\n\nprint(\"Best Epoch: {}\".format(pd_results.loc[pd_results.valid_acc.idxmax()]['epoch']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(pd_results['epoch'],pd_results['valid_loss'], label='validation_loss')\naxes[0].plot(pd_results['epoch'],pd_results['train_loss'], label='train_loss')\n# axes[0].plot(pd_results['epoch'],pd_results['test_loss'], label='test_loss')\n\naxes[0].legend()\n\naxes[1].plot(pd_results['epoch'],pd_results['valid_acc'], label='validation_acc')\naxes[1].plot(pd_results['epoch'],pd_results['train_acc'], label='train_acc')\n# axes[1].plot(pd_results['epoch'],pd_results['test_acc'], label='test_acc')\naxes[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(classes)\n\n# Use the validation set to make a confusion matrix\nnet.eval() # good habit I suppose\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Goes through the val set\nfor images, _ in val_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)\n\n# Make the confusion matrix\ncmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)\nfor i in range(len(val_labels)):\n    cmt[val_labels[i], predictions[i]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Time to get the network's predictions on the test set\n# Put the test set in a DataLoader\n\nnet.eval() # Safety first\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Go through the test set, saving the predictions in... 'predictions'\nfor images in submission_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in the sample submission\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\n\n# Change the label column to our predictions \n# Have to make sure the predictions Tensor is on the cpu\nsubmission['label'] = predictions.cpu().numpy()\n# Write the dataframe to a new csv, not including the index\nsubmission.to_csv(\"predictions.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}