{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import libraries\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nfrom torch import optim\nfrom torchvision import transforms","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Prepare Dataset\n# load data\ntrain = pd.read_csv(r\"../input/train.csv\",dtype = np.float32)\n\n# split data into features(pixels) and labels(numbers from 0 to 9)\ntargets_numpy = train.label.values\nfeatures_numpy = train.loc[:,train.columns != \"label\"].values/255 # normalization\n\n# train test split. Size of train data is 80% and size of test data is 20%. \nfeatures_train, features_test, targets_train, targets_test = train_test_split(features_numpy,\n                                                                             targets_numpy,\n                                                                             test_size = 0.2,\n                                                                             random_state = 42) \n\n# create feature and targets tensor for train set. As you remember we need variable to accumulate gradients. Therefore first we create tensor, then we will create variable\nfeaturesTrain = torch.from_numpy(features_train)\ntargetsTrain = torch.from_numpy(targets_train).type(torch.LongTensor) # data type is long\n\n# create feature and targets tensor for test set.\nfeaturesTest = torch.from_numpy(features_test)\ntargetsTest = torch.from_numpy(targets_test).type(torch.LongTensor) # data type is long\n\n# batch_size, epoch and iteration\nbatch_size = 100\nn_iters = 10000\nnum_epochs = n_iters / (len(features_train) / batch_size)\nnum_epochs = int(num_epochs)\n\n# Pytorch train and test sets\ntrain = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)\ntest = torch.utils.data.TensorDataset(featuresTest,targetsTest)\n\n# data loader\ntrain_loader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = False)\ntest_loader = torch.utils.data.DataLoader(test, batch_size = batch_size, shuffle = False)\n\n\ndataiter = iter(train_loader)\nimages, labels = dataiter.next()\nprint(type(images))\nprint(images.shape)\nprint(labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize one of the images in data set\nplt.imshow(features_numpy[10].reshape(28,28))\nplt.axis(\"off\")\nplt.title(str(targets_numpy[10]))\nplt.savefig('graph.png')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import libraries and create model architecture\nfrom torch import nn, optim\nimport torch.nn.functional as F\n\nclass Classifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.fc1 = nn.Linear(784, 256)\n        self.fc2 = nn.Linear(256, 128)\n        self.fc3 = nn.Linear(128, 64)\n        self.fc4 = nn.Linear(64, 10)\n        \n    def forward(self, x):\n        # make sure input tensor is flattened\n        x = x.view(x.shape[0], -1)\n        \n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = F.relu(self.fc3(x))\n        x = F.log_softmax(self.fc4(x), dim=1)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Classifier()\n\nimages, labels = next(iter(test_loader))\n# Get the class probabilities\nps = torch.exp(model(images))\n# Make sure the shape is appropriate, we should get 10 class probabilities for 64 examples\nprint(ps.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_p, top_class = ps.topk(1, dim=1)\n# Look at the most likely classes for the first 10 examples\nprint(top_class[:10,:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#equate topclass and labels\nequals = top_class == labels.view(*top_class.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy = torch.mean(equals.type(torch.FloatTensor))\nprint(f'Accuracy: {accuracy.item()*100}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Classifier()\ncriterion = nn.NLLLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.003)\n\nepochs = 30\nsteps = 0\n\ntrain_losses, test_losses = [], []\nfor e in range(epochs):\n    running_loss = 0\n    for images, labels in train_loader:\n        \n        optimizer.zero_grad()\n        \n        log_ps = model(images)\n        loss = criterion(log_ps, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        \n    else:\n        test_loss = 0\n        accuracy = 0\n        \n        # Turn off gradients for validation, saves memory and computations\n        with torch.no_grad():\n            for images, labels in test_loader:\n                log_ps = model(images)\n                test_loss += criterion(log_ps, labels)\n                \n                ps = torch.exp(log_ps)\n                top_p, top_class = ps.topk(1, dim=1)\n                equals = top_class == labels.view(*top_class.shape)\n                accuracy += torch.mean(equals.type(torch.FloatTensor))\n                \n        train_losses.append(running_loss/len(train_loader))\n        test_losses.append(test_loss/len(test_loader))\n\n        print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n              \"Training Loss: {:.3f}.. \".format(running_loss/len(train_loader)),\n              \"Test Loss: {:.3f}.. \".format(test_loss/len(test_loader)),\n              \"Test Accuracy: {:.3f}\".format(accuracy/len(test_loader)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_losses, label='Training loss')\nplt.plot(test_losses, label='Validation loss')\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we can clearly see that our model is overfitting. So, let's add dropout to our model to lower validation loss."},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.fc1 = nn.Linear(784, 256)\n        self.fc2 = nn.Linear(256, 128)\n        self.fc3 = nn.Linear(128, 64)\n        self.fc4 = nn.Linear(64, 10)\n\n        # Dropout module with 0.2 drop probability\n        self.dropout = nn.Dropout(p=0.2)\n\n    def forward(self, x):\n        # make sure input tensor is flattened\n        x = x.view(x.shape[0], -1)\n\n        # Now with dropout\n        x = self.dropout(F.relu(self.fc1(x)))\n        x = self.dropout(F.relu(self.fc2(x)))\n        x = self.dropout(F.relu(self.fc3(x)))\n\n        # output so no dropout here\n        x = F.log_softmax(self.fc4(x), dim=1)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Classifier()\ncriterion = nn.NLLLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\nepochs = 30\nsteps = 0\n\ntrain_losses, test_losses = [], []\nfor e in range(epochs):\n    running_loss = 0\n    for images, labels in train_loader:\n        \n        optimizer.zero_grad()\n        \n        log_ps = model(images)\n        loss = criterion(log_ps, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        \n    else:\n        test_loss = 0\n        accuracy = 0\n        \n        # Turn off gradients for validation, saves memory and computations\n        with torch.no_grad():\n            model.eval()\n            for images, labels in test_loader:\n                log_ps = model(images)\n                test_loss += criterion(log_ps, labels)\n                \n                ps = torch.exp(log_ps)\n                top_p, top_class = ps.topk(1, dim=1)\n                equals = top_class == labels.view(*top_class.shape)\n                accuracy += torch.mean(equals.type(torch.FloatTensor))\n        \n        model.train()\n        \n        train_losses.append(running_loss/len(train_loader))\n        test_losses.append(test_loss/len(test_loader))\n\n        print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n              \"Training Loss: {:.3f}.. \".format(running_loss/len(train_loader)),\n              \"Test Loss: {:.3f}.. \".format(test_loss/len(test_loader)),\n              \"Test Accuracy: {:.3f}\".format(accuracy/len(test_loader)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_losses, label='Training loss')\nplt.plot(test_losses, label='Validation loss')\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(img,ps):\n    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze())\n    ax1.axis('off')\n    ps = ps.data.numpy().squeeze()\n    \n    ax2.barh(np.arange(10), ps)\n    ax2.set_aspect(0.1)\n    ax2.set_yticks(np.arange(10))\n    ax2.set_yticklabels(np.arange(10))\n    ax2.set_title('Class Probability')\n    ax2.set_xlim(0, 1.1)\n    \n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, labels = next(iter(train_loader))\n\nimg = images[0].view(1, 784)\n# Turn off gradients to speed up this part\nwith torch.no_grad():\n    logps = model.forward(img)\n\n# Output of the network are logits, need to take softmax for probabilities\nps = torch.exp(logps)\nplot(img,ps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#predictions of first 10 images\nps = torch.exp(model(images))\ntop_p, top_class = ps.topk(1,dim=1)\nprint(top_class[:10,:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#equate topclass and labels\nequals = top_class == labels.view(*top_class.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy = torch.mean(equals.type(torch.FloatTensor))\nprint(f'Accuracy: {accuracy.item()*100}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}