{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport os\nimport torch\nimport torchvision\nfrom torchvision.utils import make_grid\nfrom torch.utils.data import random_split\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom torch.utils.data.dataloader import DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if torch.cuda.is_available():\n    device = torch.device('cuda')\nelse:\n    device = torch.device('cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np\n\n# Dataset Transformation Class\nclass MNISTDataset:\n    def __init__(self, data_path, transforms = None):\n        self.data = pd.read_csv(data_path)\n        print('Dataset Shape is : ',self.data.shape)\n        self.transform = transforms\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        # load image as ndarray type (Height * Width * Channels)\n        # be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]\n        # in this example, i don't use ToTensor() method of torchvision.transforms\n        # so you can convert numpy ndarray shape to tensor in PyTorch (H, W, C) --> (C, H, W)\n        image = self.data.iloc[index, 1:].values.astype(np.uint8).reshape((28, 28, 1))\n        label = self.data.iloc[index, 0]\n        \n        # Checking if Transform is null or not\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label\n\nclass ToTensor:\n    def __call__(self, image):\n        return torch.from_numpy(image)\n\n    \n# Apply Transformation on the Train and Test Data\ntransforms = torchvision.transforms.ToTensor()\n\n# Loading the Training and Test Dataset\ntrain_data = MNISTDataset('../input/digit-recognizer/train.csv', transforms = transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking the Size of the Datasets\nprint(len(train_data))\nsample = train_data[0]\nimg, label = sample\nprint(type(img), type(label))\nprint('Image Size : ', img.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generating the seed in order to remove Randomization\n\ntorch.manual_seed(43)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating Validation and Training Split\n\nval_size = 5000\ntrain_size = len(train_data) - val_size\n\ntraindata, valdata = random_split(train_data, [train_size, val_size])\nprint(len(traindata), len(valdata))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating Batches of Training Data\n\ntrain_loader = DataLoader(traindata, batch_size = 128, shuffle = True, num_workers = 4, pin_memory = True)\nval_loader = DataLoader(valdata, batch_size = 128, shuffle = True, num_workers = 4, pin_memory = True)\n\ntrain_iter = iter(train_loader)\nimgs, labels = train_iter.next()\nprint('Batch Images Size is : ',imgs.size())\nprint('Batch Labels Size is : ',labels.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualizing batch of Training Images using make_grid\n\nfor images, labels in train_loader:\n    plt.figure(figsize = (16,8))\n    plt.axis('off')\n    plt.imshow(make_grid(images, nrow = 16).permute((1,2,0)))\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Committing upto this point into the Jovian\n!pip install jovian --upgrade -q\nimport jovian\nproject_name = 'digit_recognizer_feed_forward'\njovian.commit(project = project_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Logging the necessary info\njovian.log_dataset(dataset_url = 'https://www.kaggle.com/c/digit-recognizer/data', val_size = val_size, random_seed = 43)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL RELATED FUNCTIONS"},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy_function(preds, labels):\n    _, indices = torch.max(preds, dim = 1)\n    return torch.tensor(torch.sum(indices == labels).item() / len(labels))\n\ndef predict(preds):\n    \n    return indices\n\nclass NNBaseClass(nn.Module):\n    def __init__(self, input_size, hidden_size1, hidden_size2, output_size):\n        super().__init__()\n        self.network = nn.Sequential(\n            nn.Linear(input_size, hidden_size1),\n            nn.ReLU(),\n            nn.Linear(hidden_size1, hidden_size2),\n            nn.ReLU(),\n            nn.Linear(hidden_size2, output_size))\n        \n    def forward(self, X):\n        X = X.view(X.size(0), -1)\n        return self.network(X)\n    \n    def prediction(self, batch):\n        img, labels = batch\n        # Moving batches into the Device\n        img = img.to(device)\n        labels = labels.to(device)\n        outputs = self.forward(img)\n        _, indices = torch.max(outputs, dim = 1)\n        return indices\n    \n    \n    def training_batch(self, batch):\n        img, labels = batch\n        # Moving batches into the Device\n        img = img.to(device)\n        labels = labels.to(device)\n        outputs = self.forward(img)\n        loss = F.cross_entropy(outputs, labels)\n        return loss\n    \n    def validating_batch(self, batch):\n        img, labels = batch\n        # Moving batches into the Device\n        img = img.to(device)\n        labels = labels.to(device)\n        outputs = self.forward(img)\n        loss = F.cross_entropy(outputs, labels)\n        acc = accuracy_function(outputs, labels)\n        return {'val_loss': loss, 'val_acc': acc}\n    \n    def validation_log(self, val_dict):\n        # Extracting Loss and Accuracy from the Val_Dict\n        losses = [x['val_loss'] for x in val_dict]\n        accuracies = [x['val_acc'] for x in val_dict]\n        mean_loss  = torch.stack(losses).mean()\n        mean_acc   = torch.stack(accuracies).mean()\n        return {'val_batch_loss': mean_loss.item(), 'val_batch_acc': mean_acc.item()}\n\n    def epoch_log(self,curr_epoch, total_epochs, model_dict):\n        print(f\"Epoch : [{curr_epoch}/{total_epochs}], Training Loss : {model_dict['train_batch_loss']}, Val Loss : {model_dict['val_batch_loss']}, Val Acc : {model_dict['val_batch_acc']}\")\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyperparameters Setting\ninput_size = 784\nhidden_size1 = 40\nhidden_size2 = 30\noutput_size = 10\n\nmodel = NNBaseClass(input_size, hidden_size1, hidden_size2, output_size).to(device)\n\nlr = 0.001\noptimizer = torch.optim.Adam(model.parameters(), lr = lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Inspecting the parameters of the Model\nfor params in model.parameters():\n    print(params.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Doing the initial prediction from the randomly initialize model\nfor images, labels in train_loader:\n    images = images.to(device)\n    labels = labels.to(device)\n    outputs = model.forward(images)\n    loss = F.cross_entropy(outputs, labels)\n    acc = accuracy_function(outputs, labels)\n    break\nprint('Loss of the Batch is : ',loss.item())\nprint('Accuracy of the Batch is : ', acc.item())\nprint('Outputs Tensor is : ', outputs[:2].data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Committing and Logging into Jovian\n\njovian.reset()\njovian.log_hyperparams({\n    'num_epochs': 20,\n    'batch_size': 128,\n    'lr': lr\n})\n\njovian.commit(project = project_name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"@torch.no_grad()\ndef Evaluate_Func(model, val_loader1):\n    logs = [model.validating_batch(batch) for batch in val_loader1]\n    return model.validation_log(logs)\n\ndef Training_Func(model, num_epochs, train_loader, val_loader, optimizer):\n    history = []\n    for epoch in range(num_epochs):\n        train_history = []\n        for batch in train_loader:\n            loss = model.training_batch(batch)\n            train_history.append(loss)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n            \n        with torch.no_grad():   \n            result = Evaluate_Func(model, val_loader)\n            result['train_batch_loss'] = torch.stack(train_history).mean().item()\n            model.epoch_log(epoch, num_epochs, result)\n            history.append(result)\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = [Evaluate_Func(model, val_loader)]\nresult","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 20\nhistory = Training_Func(model, num_epochs, train_loader, val_loader, optimizer)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotting the Results of the Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Validation Accuracy\n\nacc = [x['val_batch_acc'] for x in history]\nplt.plot(acc, '-rx')\nplt.xlabel('Number of Epochs')\nplt.ylabel('Validation Accuracy')\nplt.title('Validation Acc w.r.t Epochs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Loss Vs Validation Loss\n\ntrain_loss = [x['train_batch_loss'] for x in history]\nval_loss   = [x['val_batch_loss'] for x in history]\nplt.plot(train_loss, '-bx')\nplt.plot(val_loss, '-rx')\nplt.xlabel('Number of Epochs')\nplt.ylabel('Loss')\nplt.title('Loss w.r.t Epochs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jovian.log_metrics(train_loss=history[-1]['train_batch_loss'], \n                   val_loss=history[-1]['val_batch_loss'], \n                   val_acc=history[-1]['val_batch_acc'])\njovian.commit(project = project_name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Handling Test Data "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test Dataset Transformation Class\nclass TestMNISTDataset:\n    def __init__(self, data_path, transforms = None):\n        self.data = pd.read_csv(data_path)\n        print('Dataset Shape is : ',self.data.shape)\n        self.transform = transforms\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        # load image as ndarray type (Height * Width * Channels)\n        # be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]\n        # in this example, i don't use ToTensor() method of torchvision.transforms\n        # so you can convert numpy ndarray shape to tensor in PyTorch (H, W, C) --> (C, H, W)\n        image = self.data.iloc[index,:].values.astype(np.uint8).reshape((28, 28, 1))\n        \n        # Checking if Transform is null or not\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label\n    \nclass ToTensor:\n    def __call__(self, image):\n        return torch.from_numpy(image)\n\n\ntest_data = TestMNISTDataset('../input/digit-recognizer/test.csv', transforms = transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking the Size of the Datasets\n\nprint(len(test_data))\nsample = test_data[0]\nimg, label = sample\nprint(type(img), type(label))\nprint('Image Size : ', img.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loader = DataLoader(test_data, batch_size = 128, num_workers = 4, pin_memory = True)\ntest_iter = iter(test_loader)\nimgs, labels = test_iter.next()\nprint(imgs.size(), labels.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Testing TEST DATA \nindices = []\nfor batch in test_loader:\n    outputs = model.prediction(batch)\n    indices.append(outputs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Label = [] \nfor i in range(len(indices)):\n    for j in indices[i]:\n        Label.append(j.item())\nlen(Label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Writing the Submission.csv File\nsample_submission = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\nsample_submission = sample_submission.drop([\"Label\"], axis=1) \nsample_submission.insert(1,'Label',Label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.to_csv('Digit_Recognizer_Submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model Checkpoint Saving\n\ncheckpoint = {'model_state_dict':model.state_dict(), 'optimizer.state_dict':optimizer.state_dict()}\nFILE = 'MNIST_CHECKPOINT.pth'\ntorch.save(checkpoint, FILE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Commiting the Code to Jovian \n\njovian.commit(project =  project_name, environment = None, outputs = [checkpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}