{"cells":[{"metadata":{},"cell_type":"markdown","source":"# ResNet Kanada - Pytorch\n_October 1st 2019_\n\n**Kernels:**\n1. [Dense Net](https://www.kaggle.com/nicapotato/dense-digit-classifier-kanada-simple-cpu-pytorch)\n2. [Convolutional Neural Net](https://www.kaggle.com/nicapotato/pytorch-cnn-kanada)\n3. [ResNet50 Neural Net]()\n\n**Aim:** <br>\nBuild upon my simple dense net by introducing convolutional layers and well as some image pre-processing.\n\n**Additions:** <br>\n1. Implement ResNet\n2. CallBacks:\n    - Early Stopping\n    - Weight Decay"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# Any results you write to the current directory are saved as output.\n\nimport torch\nimport torchvision\nfrom torchvision import transforms, datasets\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\nimport math\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data\ntrain=pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest=pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')\nsubmission_set = pd.read_csv(\"../input/Kannada-MNIST/test.csv\").iloc[:,1:]\n\ntrain_data=train.drop('label',axis=1)\ntrain_targets=train['label']\n\ntest_images=test.drop('label',axis=1)\ntest_labels=test['label']\n\n# Train Test Split\ntrain_images, val_images, train_labels, val_labels = train_test_split(train_data, \n                                                                     train_targets, \n                                                                     test_size=0.2)\n\n# Reset Index\ntrain_images.reset_index(drop=True, inplace=True)\ntrain_labels.reset_index(drop=True, inplace=True)\n\nval_images.reset_index(drop=True, inplace=True)\nval_labels.reset_index(drop=True, inplace=True)\n\ntest_images.reset_index(drop=True, inplace=True)\ntest_labels.reset_index(drop=True, inplace=True)\n\nprint(\"Train Set\")\nprint(train_images.shape)\nprint(train_labels.shape)\n\nprint(\"Validation Set\")\nprint(val_images.shape)\nprint(val_labels.shape)\n\nprint(\"Validation 2\")\nprint(test_images.shape)\nprint(test_labels.shape)\n\nprint(\"Submission\")\nprint(submission_set.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Look at image means\")\nprint(train_images.mean(axis = 1).mean())\nprint(val_images.mean(axis = 1).mean())\nprint(test_images.mean(axis = 1).mean())\nprint(submission_set.mean(axis = 1).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Distribution\")\nprint(train_labels.value_counts(normalize = True))\n\nprint(\"\\nSubmission Distribution\")\nprint(test_labels.value_counts(normalize = True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMGSIZE = 28\n\n# Transformations for the train\ntrain_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.RandomCrop(IMGSIZE),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1)),\n    transforms.ToTensor(), # divides by 255\n  #  transforms.Normalize((0.5,), (0.5,))\n]))\n\n# Transformations for the validation & test sets\nval_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.ToTensor(), # divides by 255\n   # transforms.Normalize((0.1307,), (0.3081,))\n]))\n\nclass KannadaDataSet(torch.utils.data.Dataset):\n    def __init__(self, images, labels,transforms = None):\n        self.X = images\n        self.y = labels\n        self.transforms = transforms\n         \n    def __len__(self):\n        return (len(self.X))\n    \n    def __getitem__(self, i):\n        data = self.X.iloc[i,:]\n        data = np.array(data).astype(np.uint8).reshape(IMGSIZE,IMGSIZE,1)\n        \n        if self.transforms:\n            data = self.transforms(data)\n            \n        if self.y is not None:\n            return (data, self.y[i])\n        else:\n            return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\ntrain_data = KannadaDataSet(train_images, train_labels, train_trans)\nval_data = KannadaDataSet(val_images, val_labels, val_trans)\ntest_data = KannadaDataSet(test_images, test_labels, val_trans)\nsubmission_data = KannadaDataSet(submission_set, None, val_trans)\n\n\ntrain_loader = torch.utils.data.DataLoader(train_data, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\nval_loader = torch.utils.data.DataLoader(val_data, \n                                           batch_size=batch_size, \n                                           shuffle=False)\n\ntest_loader = torch.utils.data.DataLoader(test_data,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_data,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def myResNet():\n    net = torchvision.models.resnet50()\n    # First Layer\n    net.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    net.fc =  nn.Linear(in_features=2048, out_features=124, bias=True)\n    net.relufc = nn.ReLU()\n    # Finally Layer\n    net.out =  nn.Linear(in_features=124, out_features=10, bias=True)\n    \n    return net.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = myResNet()\nnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 14\nnn_output = []\n\noptimizer = optim.Adam(net.parameters(), lr=1e-3)\ncriterion = nn.CrossEntropyLoss()\n\ndef get_num_correct(preds, labels):\n    return preds.argmax(dim=1).eq(labels).sum().item()\n\nfor epoch in range(EPOCHS):\n    epoch_loss = 0\n    epoch_correct = 0\n    net.train()\n    \n    for data in train_loader:\n        # `data` is a batch of data\n        # Before using transforms, I used .unsqueeze(1) to enter a empty number channel array (Batch, Number Channels, height, width).\n        X = data[0].to(device) # X is the batch of features\n        # Unsqueeze adds a placeholder dimension for the color channel - (8, 28, 28) to (8, 1, 28, 28)\n        y = data[1].to(device) # y is the batch of targets.\n        \n        net.zero_grad()  # sets gradients to 0 before loss calc. You will do this likely every step.\n        output = net(X)  # pass in the reshaped batch (recall they are 28x28 atm)\n        tloss = criterion(output, y)  # calc and grab the loss value\n        tloss.backward()  # apply this loss backwards thru the network's parameters\n        optimizer.step()  # attempt to optimize weights to account for loss/gradients \n        \n        epoch_loss += tloss.item()\n        epoch_correct += get_num_correct(output, y)\n    \n    # Evaluation with the validation set\n    net.eval() # eval mode\n    val_loss = 0\n    val_correct = 0\n    test_loss = 0\n    test_correct = 0\n    \n    with torch.no_grad():\n        # First Validation Set\n        for data in val_loader:\n            X = data[0].to(device)\n            y = data[1].to(device)\n            \n            preds = net(X) # get predictions\n            vloss = criterion(preds, y) # calculate the loss\n            \n            val_correct += get_num_correct(preds, y)\n            val_loss += vloss.item()\n        \n        # Second Validation Set..\n        for data in test_loader:\n            X = data[0].to(device)\n            y = data[1].to(device)\n            \n            preds = net(X) # get predictions\n            tstloss = criterion(preds, y) # calculate the loss\n            \n            test_correct += get_num_correct(preds, y)\n            test_loss += tstloss.item()\n    \n    tmp_nn_output = [epoch + 1,EPOCHS,\n                     epoch_loss/len(train_loader.dataset),epoch_correct/len(train_loader.dataset)*100,\n                     val_loss/len(val_loader.dataset), val_correct/len(val_loader.dataset)*100,\n                     test_loss/len(test_loader.dataset), test_correct/len(test_loader.dataset)*100\n                    ]\n    nn_output.append(tmp_nn_output)\n    \n    # Print the loss and accuracy for the validation set\n    print('Epoch [{}/{}] train loss: {:.6f} acc: {:.3f} - valid loss: {:.6f} acc: {:.3f} - Test loss: {:.6f} acc: {:.3f}'\n        .format(*tmp_nn_output))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_results = pd.DataFrame(nn_output,\n    columns = ['epoch','total_epochs','train_loss','train_acc','valid_loss','valid_acc','test_loss','test_acc']\n                         )\ndisplay(pd_results)\n\nprint(\"Best Epoch: {}\".format(pd_results.loc[pd_results.valid_acc.idxmax()]['epoch']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(pd_results['epoch'],pd_results['valid_loss'], label='validation_loss')\naxes[0].plot(pd_results['epoch'],pd_results['train_loss'], label='train_loss')\n# axes[0].plot(pd_results['epoch'],pd_results['test_loss'], label='test_loss')\n\naxes[0].legend()\n\naxes[1].plot(pd_results['epoch'],pd_results['valid_acc'], label='validation_acc')\naxes[1].plot(pd_results['epoch'],pd_results['train_acc'], label='train_acc')\n# axes[1].plot(pd_results['epoch'],pd_results['test_acc'], label='test_acc')\naxes[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(classes)\n\n# Use the validation set to make a confusion matrix\nnet.eval() # good habit I suppose\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Goes through the val set\nfor images, _ in val_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)\n\n# Make the confusion matrix\ncmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)\nfor i in range(len(val_labels)):\n    cmt[val_labels[i], predictions[i]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Time to get the network's predictions on the test set\n# Put the test set in a DataLoader\n\nnet.eval() # Safety first\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Go through the test set, saving the predictions in... 'predictions'\nfor images in submission_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in the sample submission\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\n\n# Change the label column to our predictions \n# Have to make sure the predictions Tensor is on the cpu\nsubmission['label'] = predictions.cpu().numpy()\n# Write the dataframe to a new csv, not including the index\nsubmission.to_csv(\"predictions.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}