{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Dense Digit Classifier Kanada - Simple CPU PyTorch\n_October 1st 2019_\n\n**Kernels:**\n1. [Dense Net](https://www.kaggle.com/nicapotato/dense-digit-classifier-kanada-simple-cpu-pytorch)\n2. [Convolutional Neural Net](https://www.kaggle.com/nicapotato/pytorch-cnn-kanada)\n\n**Aim:** <br>\nExperiment with the simplest computer vision problem, image classification, using a dense feedforward neural network. No convolution used."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import transforms, datasets\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data\ntrain=pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest=pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')\nsubmission_set = pd.read_csv(\"../input/Kannada-MNIST/test.csv\").iloc[:,1:]\n\ntrain_data=train.drop('label',axis=1)\ntrain_targets=train['label']\n\ntest_images=test.drop('label',axis=1)\ntest_labels=test['label']\n\n# Train Test Split\ntrain_images, val_images, train_labels, val_labels = train_test_split(train_data, \n                                                                     train_targets, \n                                                                     test_size=0.2)\n\n# Reset Index\ntrain_images.reset_index(drop=True, inplace=True)\nval_images.reset_index(drop=True, inplace=True)\ntrain_labels.reset_index(drop=True, inplace=True)\nval_labels.reset_index(drop=True, inplace=True)\n\nprint(\"Train Set\")\nprint(train_images.shape)\nprint(train_labels.shape)\n\nprint(\"Validation Set\")\nprint(val_images.shape)\nprint(val_labels.shape)\n\nprint(\"Validation 2\")\nprint(test_images.shape)\nprint(test_labels.shape)\n\nprint(\"Submission\")\nprint(submission_set.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Distribution\")\nprint(train_labels.value_counts(normalize = True))\n\nprint(\"\\nSubmission Distribution\")\nprint(test_labels.value_counts(normalize = True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train\ntrain_data=torch.from_numpy(train_images.values).float().view(train_images.shape[0],28,28)\ntrain_targets=torch.from_numpy(train_labels.values).long().view(train_labels.shape[0])\n\n# Validation\nval_data=torch.from_numpy(val_images.values).float().view(val_images.shape[0],28,28)\nval_targets=torch.from_numpy(val_labels.values).long().view(val_labels.shape[0])\n\n# Test\ntest_data=torch.from_numpy(test_images.values).float().view(test_images.shape[0],28,28)\ntest_targets=torch.from_numpy(test_labels.values).long().view(test_labels.shape[0])\n\nsubmission_data=torch.from_numpy(submission_set.values).float().view(submission_set.shape[0],28,28)\n\nprint(train_targets.shape)\nprint(train_data.shape)\n\nprint(val_targets.shape)\nprint(val_data.shape)\n\nprint(test_targets.shape)\nprint(test_data.shape)\n\nprint(submission_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Data\")\nfig, ax = plt.subplots(nrows=5, ncols=5, figsize=(10,10))\nfig.subplots_adjust(hspace=.3)\nfor i in range(5):\n    for j in range(5):\n        rand_int = np.random.randint(train_data.shape[0])\n        ax[i][j].axis('off')\n        ax[i][j].imshow(train_data[rand_int])\n        ax[i][j].set_title(train_targets[rand_int].item())\nplt.show()\n\nprint(\"Valid 1 Data\")\nfig, ax = plt.subplots(nrows=5, ncols=5, figsize=(10,10))\nfig.subplots_adjust(hspace=.3)\nfor i in range(5):\n    for j in range(5):\n        rand_int = np.random.randint(val_data.shape[0])\n        ax[i][j].axis('off')\n        ax[i][j].imshow(val_data[rand_int])\n        ax[i][j].set_title(val_targets[rand_int].item())\nplt.show()\n\nprint(\"Valid 2 Data\")\nfig, ax = plt.subplots(nrows=5, ncols=5, figsize=(10,10))\nfig.subplots_adjust(hspace=.3)\nfor i in range(5):\n    for j in range(5):\n        rand_int = np.random.randint(test_data.shape[0])\n        ax[i][j].axis('off')\n        ax[i][j].imshow(test_data[rand_int])\n        ax[i][j].set_title(test_targets[rand_int].item())\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Submission Data\")\nfig, ax = plt.subplots(nrows=5, ncols=5, figsize=(10,10))\nfig.subplots_adjust(hspace=.3)\nfor i in range(5):\n    for j in range(5):\n        rand_int = np.random.randint(submission_data.shape[0])\n        ax[i][j].axis('off')\n        ax[i][j].imshow(submission_data[rand_int], cmap='viridis')\n#         ax[i][j].set_title(train_targets[rand_int].item())\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=10, ncols=10, figsize=(10,10))\n\n# I know these for loops look weird, but this way num_i is only computed once for each class\nfor i in range(10): # Column by column\n    num_i = train_images[train_labels == i]\n    ax[0][i].set_title(i)\n    for j in range(10): # Row by row\n        ax[j][i].axis('off')\n        ax[j][i].imshow(num_i.iloc[j, :].to_numpy().astype(np.uint8).reshape(28, 28), cmap='viridis')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 8\n\ntrain_set=torch.utils.data.TensorDataset(train_data, train_targets)\nvalid_set=torch.utils.data.TensorDataset(val_data, val_targets)\ntest_set=torch.utils.data.TensorDataset(test_data, test_targets)\nsubmission_set=torch.utils.data.TensorDataset(submission_data) \n\ntrain_loader = torch.utils.data.DataLoader(train_set, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\nval_loader = torch.utils.data.DataLoader(valid_set, \n                                           batch_size=batch_size, \n                                           shuffle=False)\n\ntest_loader = torch.utils.data.DataLoader(test_set,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_set,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, dropout = .05):\n        super().__init__()\n        self.dropout = dropout\n        \n        self.fc1 = nn.Linear(28*28, 64)\n        self.d1 = nn.Dropout(p= self.dropout)\n        self.fc2 = nn.Linear(64, 128)\n        self.d2 = nn.Dropout(p= self.dropout)\n        self.fc3 = nn.Linear(128, 64)\n        self.d3 = nn.Dropout(p= self.dropout)\n        self.fc4 = nn.Linear(64, 10)\n\n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = self.d1(x)\n        x = F.relu(self.fc2(x))\n        x = self.d2(x)\n        x = F.relu(self.fc3(x))\n        x = self.d3(x)\n        x = self.fc4(x)\n        return F.log_softmax(x, dim=1)\n\nnet = Net()\nnet.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 13\n\nloss_function = nn.CrossEntropyLoss()\noptimizer = optim.Adam(net.parameters(), lr=0.001)\n\nnn_output = []\n\ndef get_num_correct(preds, labels):\n    return preds.argmax(dim=1).eq(labels).sum().item()\n\nfor epoch in range(EPOCHS): # 3 full passes over the data\n    epoch_loss = 0\n    epoch_correct = 0\n    net.train()\n    \n    for data in train_loader:  # `data` is a batch of data\n        X, y = data[0].to(device), data[1].to(device)  # X is the batch of features, y is the batch of targets.\n        net.zero_grad()  # sets gradients to 0 before loss calc. You will do this likely every step.\n        output = net(X.view(-1,784))  # pass in the reshaped batch (recall they are 28x28 atm)\n        tloss = F.nll_loss(output, y)  # calc and grab the loss value\n        tloss.backward()  # apply this loss backwards thru the network's parameters\n        optimizer.step()  # attempt to optimize weights to account for loss/gradients \n        \n        epoch_loss += tloss.item()\n        epoch_correct += get_num_correct(output, y)\n    \n    # Evaluation with the validation set\n    net.eval() # eval mode\n    val_loss = 0\n    val_correct = 0\n    with torch.no_grad():\n        for data in val_loader:\n            X, y = data[0].to(device), data[1].to(device)\n            \n            preds = net(X.view(-1,784)) # get predictions\n            vloss = F.cross_entropy(preds, y) # calculate the loss\n            \n            val_correct += get_num_correct(preds, y)\n            val_loss += vloss.item()\n    \n    tmp_nn_output = [epoch + 1,EPOCHS,\n                     epoch_loss/len(train_loader.dataset),epoch_correct/len(train_loader.dataset)*100,\n                     val_loss/len(val_loader.dataset), val_correct/len(val_loader.dataset)*100]\n    nn_output.append(tmp_nn_output)\n    \n    # Print the loss and accuracy for the validation set\n    print('Epoch [{}/{}] train loss: {:.4f} acc: {:.4f} valid loss: {:.4f} acc: {:.4f}'\n        .format(*tmp_nn_output))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_results = pd.DataFrame(nn_output, columns = ['epoch','total_epochs','train_loss','train_acc','valid_loss','valid_acc'])\npd_results.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(pd_results['epoch'],pd_results['valid_loss'], label='validation_loss')\naxes[0].plot(pd_results['epoch'],pd_results['train_loss'], label='train_loss')\n\naxes[1].plot(pd_results['epoch'],pd_results['valid_acc'], label='validation_acc')\naxes[1].plot(pd_results['epoch'],pd_results['train_acc'], label='train_acc')\naxes[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(classes)\n\n# Use the validation set to make a confusion matrix\nnet.eval() # good habit I suppose\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Goes through the val set\nfor images, _ in val_loader:\n    preds = net(images.view(-1,784).to(device))\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)\n\n# Make the confusion matrix\ncmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)\nfor i in range(len(val_labels)):\n    cmt[val_labels[i], predictions[i]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Time to get the network's predictions on the test set\n# Put the test set in a DataLoader\n\nnet.eval() # Safety first\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Go through the test set, saving the predictions in... 'predictions'\nfor images in submission_loader:\n    preds = net(images[0].view(-1,784).to(device))\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in the sample submission\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\n\n# Change the label column to our predictions \n# Have to make sure the predictions Tensor is on the cpu\nsubmission['label'] = predictions.cpu().numpy()\n# Write the dataframe to a new csv, not including the index\nsubmission.to_csv(\"predictions.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}