{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nfrom torch import nn, optim\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torchvision import transforms, models\nimport matplotlib.pyplot as plt\n\n\nimport pandas as pd\nimport numpy as np\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking GPU is available\ntrain_on_gpu = torch.cuda.is_available()\n\nif not train_on_gpu:\n    print('Training on CPU...')\nelse:\n    print('Training on GPU...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset responsible for manipulating data for training as well as training tests.\nclass DatasetMNIST(torch.utils.data.Dataset):\n    def __init__(self, data, transform=None):\n        self.data = data\n        print(len(self.data))\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        item = self.data.iloc[index]\n                \n        image = item[1:].values.astype(np.uint8).reshape((28, 28))\n        label = item[0]\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Modeling and Creating Data for Training and Validation"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"BATCH_SIZE = 100\nVALID_SIZE = 0.15 # percentage of data for validation\n\ntransform_train = transforms.Compose([\n    transforms.ToPILImage(),\n   # transforms.RandomRotation(0, 0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.5,), std=(0.5,))\n])\n\ntransform_valid = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.5,), std=(0.5,))\n])\n\n# Importing data that will be used for training and validation\ndataset = pd.read_csv('../input/train.csv')\n\n# Creating datasets for training and validation\ntrain_data = DatasetMNIST(dataset, transform=transform_train)\nvalid_data = DatasetMNIST(dataset, transform=transform_valid)\n\n# Shuffling data and choosing data that will be used for training and validation\nnum_train = len(train_data)\nindices = list(range(num_train))\nnp.random.shuffle(indices)\nsplit = int(np.floor(VALID_SIZE * num_train))\ntrain_idx, valid_idx = indices[split:], indices[:split]\n\ntrain_sampler = SubsetRandomSampler(train_idx)\nvalid_sampler = SubsetRandomSampler(valid_idx)\n\ntrain_loader = torch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE, sampler=train_sampler)\nvalid_loader = torch.utils.data.DataLoader(valid_data, batch_size=BATCH_SIZE, sampler=valid_sampler)\n\nprint(f\"Length train: {len(train_idx)}\")\nprint(f\"Length valid: {len(valid_idx)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Viewing data examples used for training\nfig, axis = plt.subplots(3, 10, figsize=(15, 10))\nimages, labels = next(iter(train_loader))\n\nfor i, ax in enumerate(axis.flat):\n    with torch.no_grad():\n        image, label = images[i], labels[i]\n\n        ax.imshow(image.view(28, 28), cmap='binary') # add image\n        ax.set(title = f\"{label}\") # add label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Viewing data examples used for validation\nfig, axis = plt.subplots(3, 10, figsize=(15, 10))\nimages, labels = next(iter(valid_loader))\n\nfor i, ax in enumerate(axis.flat):\n    with torch.no_grad():\n        image, label = images[i], labels[i]\n\n        ax.imshow(image.view(28, 28), cmap='binary') # add image\n        ax.set(title = f\"{label}\") # add label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Modeling and Creating Network (CNN)"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        \n        self.conv1 = nn.Sequential(\n            nn.Conv2d(1, 32, 3, padding=1),\n            nn.ReLU(),\n            nn.BatchNorm2d(32),\n            nn.Conv2d(32, 32, 3, stride=2, padding=1),\n            nn.ReLU(),\n            nn.BatchNorm2d(32),\n            nn.MaxPool2d(2, 2),\n            nn.Dropout(0.25)\n        )\n        \n        self.conv2 = nn.Sequential(\n            nn.Conv2d(32, 64, 3, padding=1),\n            nn.ReLU(),\n            nn.BatchNorm2d(64),\n            nn.Conv2d(64, 64, 3, stride=2, padding=1),\n            nn.ReLU(),\n            nn.BatchNorm2d(64),\n            nn.MaxPool2d(2, 2),\n            nn.Dropout(0.25)\n        )\n        \n        self.conv3 = nn.Sequential(\n            nn.Conv2d(64, 128, 3, padding=1),\n            nn.ReLU(),\n            nn.BatchNorm2d(128),\n            nn.MaxPool2d(2, 2),\n            nn.Dropout(0.25)\n        )\n        \n        self.fc = nn.Sequential(\n            nn.Linear(128, 10),\n        )\n                \n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        \n        x = x.view(x.size(0), -1)\n        return self.fc(x)\n\nmodel = Net()\nprint(model)\n\nif train_on_gpu:\n    model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Configuring and Training Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"LEARNING_RATE = 0.001680\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 150\nvalid_loss_min = np.Inf\ntrain_losses, valid_losses = [], []\nhistory_accuracy = []\n\nfor e in range(1, epochs+1):\n    running_loss = 0\n\n    for images, labels in train_loader:\n        if train_on_gpu:\n            images, labels = images.cuda(), labels.cuda()\n        # Clear the gradients, do this because gradients are accumulated.\n        optimizer.zero_grad()\n        \n        # Forward pass, get our log-probabilities.\n        ps = model(images)\n\n        # Calculate the loss with the logps and the labels.\n        loss = criterion(ps, labels)\n        \n        # Turning loss back.\n        loss.backward()\n        \n        # Take an update step and few the new weights.\n        optimizer.step()\n        \n        running_loss += loss.item()\n    else:\n        valid_loss = 0\n        accuracy = 0\n        \n        # Turn off gradients for validation, saves memory and computations.\n        with torch.no_grad():\n            model.eval() # change the network to evaluation mode\n            for images, labels in valid_loader:\n                if train_on_gpu:\n                    images, labels = images.cuda(), labels.cuda()\n                # Forward pass, get our log-probabilities.\n                #log_ps = model(images)\n                ps = model(images)\n                \n                # Calculating probabilities for each class.\n                #ps = torch.exp(log_ps)\n                \n                # Capturing the class more likely.\n                _, top_class = ps.topk(1, dim=1)\n                \n                # Verifying the prediction with the labels provided.\n                equals = top_class == labels.view(*top_class.shape)\n                \n                valid_loss += criterion(ps, labels)\n                accuracy += torch.mean(equals.type(torch.FloatTensor))\n                \n        model.train() # change the network to training mode\n        \n        train_losses.append(running_loss/len(train_loader))\n        valid_losses.append(valid_loss/len(valid_loader))\n        history_accuracy.append(accuracy/len(valid_loader))\n        \n        network_learned = valid_loss < valid_loss_min\n\n        if e == 1 or e % 5 == 0 or network_learned:\n            print(f\"Epoch: {e}/{epochs}.. \",\n                  f\"Training Loss: {running_loss/len(train_loader):.3f}.. \",\n                  f\"Validation Loss: {valid_loss/len(valid_loader):.3f}.. \",\n                  f\"Test Accuracy: {accuracy/len(valid_loader):.3f}\")\n        \n        if network_learned:\n            valid_loss_min = valid_loss\n            torch.save(model.state_dict(), 'model_mtl_mnist.pt')\n            print('Detected network improvement, saving current model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Viewing training information\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport matplotlib.pyplot as plt\n\nplt.plot(train_losses, label='Training Loss')\nplt.plot(valid_losses, label='Validation Loss')\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history_accuracy, label='Validation Accuracy')\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Creating Data For Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing trained Network with better loss of validation\nmodel.load_state_dict(torch.load('model_mtl_mnist.pt'))\n\nprint(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# specify the image classes\nclasses = ['0', '1', '2', '3', '4',\n           '5', '6', '7', '8', '9']\n\n# track test loss\ntest_loss = 0.0\nclass_correct = list(0. for i in range(10))\nclass_total = list(0. for i in range(10))\n\nmodel.eval()\n# iterate over test data\nfor data, target in valid_loader:\n    # move tensors to GPU if CUDA is available\n    if train_on_gpu:\n        data, target = data.cuda(), target.cuda()\n    # forward pass: compute predicted outputs by passing inputs to the model\n    output = model(data)\n    # calculate the batch loss\n    loss = criterion(output, target)\n    # update test loss \n    test_loss += loss.item()*data.size(0)\n    # convert output probabilities to predicted class\n    _, pred = torch.max(output, 1)    \n    # compare predictions to true label\n    correct_tensor = pred.eq(target.data.view_as(pred))\n    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())\n    # calculate test accuracy for each object class\n    for i in range(BATCH_SIZE):\n        label = target.data[i]\n        class_correct[label] += correct[i].item()\n        class_total[label] += 1\n\n# average test loss\ntest_loss = test_loss/len(valid_loader.dataset)\nprint('Test Loss: {:.6f}\\n'.format(test_loss))\n\nfor i in range(10):\n    if class_total[i] > 0:\n        print('Test Accuracy of %5s: %0.4f%% (%2d/%2d)' % (\n            classes[i], class_correct[i] / class_total[i],\n            np.sum(class_correct[i]), np.sum(class_total[i])))\n    else:\n        print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))\n\nprint('\\nTest Accuracy (Overall): %2.2f%% (%2d/%2d)' % (\n    100. * np.sum(class_correct) / np.sum(class_total),\n    np.sum(class_correct), np.sum(class_total)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetSubmissionMNIST(torch.utils.data.Dataset):\n    def __init__(self, file_path, transform=None):\n        self.data = pd.read_csv(file_path)\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        image = self.data.iloc[index].values.astype(np.uint8).reshape((28, 28, 1))\n\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.5,), std=(0.5,))\n])\n\nsubmissionset = DatasetSubmissionMNIST('../input/test.csv', transform=transform)\nsubmissionloader = torch.utils.data.DataLoader(submissionset, batch_size=BATCH_SIZE, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = [['ImageId', 'Label']]\n\nwith torch.no_grad():\n    model.eval()\n    image_id = 1\n\n    for images in submissionloader:\n        if train_on_gpu:\n            images = images.cuda()\n        log_ps = model(images)\n        ps = torch.exp(log_ps)\n        top_p, top_class = ps.topk(1, dim=1)\n        \n        for prediction in top_class:\n            submission.append([image_id, prediction.item()])\n            image_id += 1\n            \nprint(len(submission) - 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import csv\n\nwith open('submission.csv', 'w') as submissionFile:\n    writer = csv.writer(submissionFile)\n    writer.writerows(submission)\n    \nprint('Submission Complete!')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}