{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Imprting the libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\nimport random\nimport math\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torchvision\nfrom torchvision import datasets, models, transforms\n\nimport torch.nn as nn\nfrom torch.nn import Linear, ReLU, CrossEntropyLoss, Sequential, Conv2d, MaxPool2d, Module, Softmax, BatchNorm2d, Dropout\n\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler, Adam, SGD\n\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\n\n# for evaluating the model\nfrom sklearn.metrics import accuracy_score\nfrom tqdm import tqdm\nimport time\nimport os\nimport copy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing the Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest=pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing the Raw Data\nAs the Data is having 785 columns which have m 28*28*1 images and their labels.\nWe are converting the data in aproper format to feed them in the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"#Getting the label column\ntrain_labels = np.array(train['label'])\n# m = No of Exaples\nm_train = train.shape[0] #m in training data\nm_test = test.shape[0]  #m in testing data\n#reshaping the long 1D vector of shape 1*784 into a 3D vector of shape 1*28*28 \ntrain_data = np.array(train.loc[:,'pixel0':]).reshape(m_train,1,28,28)\ntest_data = np.array(test.loc[:,'pixel0':]).reshape(m_test,1,28,28)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualising the Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"k = 0\nwhile k<2:\n    i = random.randint(0,42000)\n    plt.imshow(train_data[i,0,:,:])\n    plt.show()\n    print(f\"The label for the above image is {train_labels[i]}\")\n    k+=1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Splitting the data into Training, Validation and Testing Set\nWe are using 90% data for training, 5% for Validation and 5% for Testing purpose."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_data, train_labels, test_size=0.2, random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape, X_val.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Transformer\n Making a Transformer to convert our data into the PyTorch Tensors and Normalize the data.\n We are converting the data with Mean = 0.5 and STD = 0.5 across the channel"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numbers\nclass RandomRotation(object):\n\n\n    def __init__(self, degrees, resample=False, expand=False, center=None):\n        if isinstance(degrees, numbers.Number):\n            if degrees < 0:\n                raise ValueError(\"If degrees is a single number, it must be positive.\")\n            self.degrees = (-degrees, degrees)\n        else:\n            if len(degrees) != 2:\n                raise ValueError(\"If degrees is a sequence, it must be of len 2.\")\n            self.degrees = degrees\n\n        self.resample = resample\n        self.expand = expand\n        self.center = center\n\n    @staticmethod\n    def get_params(degrees):\n\n        angle = np.random.uniform(degrees[0], degrees[1])\n\n        return angle\n\n    def __call__(self, img):\n\n        \n        def rotate(img, angle, resample=False, expand=False, center=None):\n\n                \n            return img.rotate(angle, resample, expand, center)\n\n        angle = self.get_params(self.degrees)\n\n        return rotate(img, angle, self.resample, self.expand, self.center)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RandomShift(object):\n    def __init__(self, shift):\n        self.shift = shift\n        \n    @staticmethod\n    def get_params(shift):\n        hshift, vshift = np.random.uniform(-shift, shift, size=2)\n\n        return hshift, vshift \n    def __call__(self, img):\n        hshift, vshift = self.get_params(self.shift)\n        \n        return img.transform(img.size, Image.AFFINE, (1,0,hshift,0,1,vshift), resample=Image.BICUBIC, fill=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transformations to be applied on images\ntransform = transforms.Compose([transforms.ToTensor(),\n                              transforms.Normalize((0.5,), (0.5,)),\n                              ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DigitDataset(Dataset):\n\n    def __init__(self,images,labels,transfrom = transform):\n        # Initialize data, download, etc.\n        self.x_data = torch.from_numpy(images/255.) # size [n_samples, n_features]\n        self.y_data = torch.from_numpy(labels) # size [n_samples, 1]\n        self.n_samples = images.shape[0]\n\n    # support indexing such that dataset[i] can be used to get i-th sample\n    def __getitem__(self, index):\n        return self.x_data[index], self.y_data[index]\n\n    # we can call len(dataset) to return the size\n    def __len__(self):\n        return self.n_samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = DigitDataset(X_train,y_train,transforms.Compose([RandomRotation(degrees=20), RandomShift(3),\n                                                                 transforms.Normalize(mean=(0.5,), std=(0.5,)),\n                             transform]))\nval_dataset = DigitDataset(X_val,y_val,transform)\ntest_dataset = DigitDataset(X_test,y_test,transform)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Creating Dataloaders\nWe will use Batch-Size = 64"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size=64\n# defining trainloader, valloader and testloader\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True)\nval_loader = torch.utils.data.DataLoader(val_dataset, batch_size=64, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=64, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Visualising the Data-loaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"# shape of training data\ndataiter = iter(train_loader)\nimages, labels = dataiter.next()\n\nprint(images.shape)\nprint(labels.shape)\n\n# visualizing the training images\nplt.imshow(images[0].numpy().squeeze(), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shape of validation data\ndataiter = iter(val_loader)\nimages, labels = dataiter.next()\n\nprint(images.shape)\nprint(labels.shape)\n\n# visualizing the training images\nplt.imshow(images[0].numpy().squeeze(), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Checking the device type\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I am using GPU to train the model hence everything is sent to GPU via *.cuda()* command"},{"metadata":{},"cell_type":"markdown","source":"# Model Building\nOur Model is taking 1 * 28 * 28 Images as input and having the output with dimension = 10"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):    \n    def __init__(self):\n        super(Net, self).__init__()\n          \n        self.features = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(32, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(64, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n          \n        self.classifier = nn.Sequential(\n            nn.Dropout(p = 0.5),\n            nn.Linear(64 * 7 * 7, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p = 0.5),\n            nn.Linear(512, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p = 0.5),\n            nn.Linear(512, 15),\n        )\n          \n\n                \n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        \n        return x   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Defining the Optimizer, Criterion (loss function) and Learning Rate Scheduler."},{"metadata":{"trusted":true},"cell_type":"code","source":"# defining the model\nmodel = Net()\n# defining the optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.003)\n# defining the loss function\ncriterion = nn.CrossEntropyLoss()\nstep_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n# checking if GPU is available\nif torch.cuda.is_available():\n    model = model.cuda()\n    criterion = criterion.cuda()\n    \nprint(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training of the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_sizes = {}\ndataset_sizes['train'] = len(train_dataset)\ndataset_sizes['val'] = len(val_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs=50):\n    since = time.time() #Return the time in seconds since the epoch as a floating point number\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    dataloaders = {}\n    dataloaders['train'] = train_loader\n    dataloaders['val'] = val_loader\n    \n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                inputs = inputs.type(torch.cuda.FloatTensor)\n                labels = labels.to(device)\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        optimizer.zero_grad()\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel = model.to(device)\n\nmodel = train_model(model, criterion, optimizer, step_lr_scheduler, num_epochs=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Testing our model on test-data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# getting predictions on test set and measuring the performance\ncorrect_count, all_count = 0, 0\nfor images,labels in test_loader:\n  for i in range(len(labels)):\n    images = images.cuda()\n    images = images.type(torch.cuda.FloatTensor)\n    labels = labels.cuda()\n    img = images[i].view(1, 1, 28, 28)\n    with torch.no_grad():\n        logps = model(img)\n\n    \n    ps = torch.exp(logps)\n    probab = list(ps.cpu()[0])\n    pred_label = probab.index(max(probab))\n    true_label = labels.cpu()[i]\n    if(true_label == pred_label):\n      correct_count += 1\n    all_count += 1\n\nprint(\"Number Of Images Tested =\", all_count)\nprint(\"\\nModel Accuracy =\", (correct_count/all_count))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predicting for unlabelled Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test_data/255 #Normalizing the data\ntest = torch.from_numpy(test)  # Converting into Tensors\ntest = test.type(torch.cuda.FloatTensor) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n  outputs = model(test.cuda())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Outputs are the output of the final linear Layer having shape = 10"},{"metadata":{"trusted":true},"cell_type":"code","source":"ps = torch.exp(outputs)\n\n#max_value is the value of highest no. in each 10-dim vector \n#index is the index of that max value \nmax_value, index = torch.max(ps,axis=1) \n\nindex = index.cpu()\n#Converting Prediction to numpy for Submission\nprediction = index.numpy()\n\nprint(prediction.shape)\nprint(prediction[:5])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Saving the Prediction in the acceptable format "},{"metadata":{"trusted":true},"cell_type":"code","source":"k = np.arange(1,28001)\nsubmission = pd.DataFrame({\n        \"ImageId\":k ,\n        \"Label\": prediction\n\n    })\n\nsubmission.to_csv('Digit_Recognition_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}