{"cells":[{"metadata":{"_cell_guid":"56682ac8-509e-44fb-a694-4bf1082d36d2","_uuid":"427ce49fba02f44de9e3081288fca2da38e51140","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom torchvision.utils import make_grid\n\nimport math, random, numbers\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_DIR = '../input/digit-recognizer'\nBATCH_SIZE = 64\nN_EPOCHS = 50","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e224069f-8b41-4390-a6d5-09ac6379982f","_uuid":"8a346d02febaa91c6ae6073dad4a227fe38242a9"},"cell_type":"markdown","source":"## Explore the Data","execution_count":null},{"metadata":{"_cell_guid":"5b89951b-5acb-4957-b6e4-cfeb54f4a053","_uuid":"86cafb6770a21e6ed806c17c2be9897399bdf520","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(INPUT_DIR + '/train.csv')\n\nn_train = len(train_df)\nn_pixels = len(train_df.columns) - 1\nn_class = len(set(train_df['label']))\n\nprint('Number of training samples: {0}'.format(n_train))\nprint('Number of training pixels: {0}'.format(n_pixels))\nprint('Number of classes: {0}'.format(n_class))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7bf732ab-a5e7-4db0-ab30-8b3f1f89bf60","_uuid":"2c697381fd6210380e2350553bd43ca13194cff1","trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(INPUT_DIR + '/test.csv')\n\nn_test = len(test_df)\nn_pixels = len(test_df.columns)\n\nprint('Number of train samples: {0}'.format(n_test))\nprint('Number of test pixels: {0}'.format(n_pixels))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f06156b2-2118-4a3b-a648-e15b70a5982e","_uuid":"1166411bc5c17d2147e0e29feb1df8c94b0f203a"},"cell_type":"markdown","source":"## Data Loader","execution_count":null},{"metadata":{"_cell_guid":"711ee835-cb44-4243-9f7e-60c6c4301cca","_uuid":"13401816f84176712146c336d11bdab8ce549bce","trusted":true},"cell_type":"code","source":"class MNIST_data(Dataset):\n    def __init__(self, file_path, \n                 transform = transforms.Compose([transforms.ToPILImage(), transforms.ToTensor(), \n                     transforms.Normalize(mean=(0.5,), std=(0.5,))])\n                ):\n        \n        df = pd.read_csv(file_path)\n        \n        if len(df.columns) == n_pixels:\n            # test data\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            # training data\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n            \n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        if self.y is not None:\n            return self.transform(self.X[idx]), self.y[idx]\n        else:\n            return self.transform(self.X[idx])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a593e310-63dc-49a5-b939-ad6a4889b068","_uuid":"db3edd720c83473a77bcc43b463e83d87569d7bb"},"cell_type":"markdown","source":"### Random Rotation Transformation\nRandomly rotate the image. Available in upcoming torchvision but not now.","execution_count":null},{"metadata":{"_cell_guid":"2b9f5823-708a-446c-9e49-a430d6bc04cb","_uuid":"cc6c3fb42e4a98508bf910b5d289dbe5a0e78bc3","trusted":true},"cell_type":"code","source":"class RandomRotation(object):\n    def __init__(self, degrees, resample=False, expand=False, center=None):\n        if isinstance(degrees, numbers.Number):\n            if degrees < 0:\n                raise ValueError(\"If degrees is a single number, it must be positive.\")\n            self.degrees = (-degrees, degrees)\n        else:\n            if len(degrees) != 2:\n                raise ValueError(\"If degrees is a sequence, it must be of len 2.\")\n            self.degrees = degrees\n\n        self.resample = resample\n        self.expand = expand\n        self.center = center\n\n    @staticmethod\n    def get_params(degrees):\n        angle = np.random.uniform(degrees[0], degrees[1])\n        return angle\n\n    def __call__(self, img):\n        def rotate(img, angle, resample=False, expand=False, center=None):\n            return img.rotate(angle, resample, expand, center)\n        angle = self.get_params(self.degrees)\n        return rotate(img, angle, self.resample, self.expand, self.center)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8bbe7231-1109-4531-9ecc-a25e575be23a","_uuid":"97ef1e719ef91ea602e9c4ad7ccef6ed6e1957d1"},"cell_type":"markdown","source":"### Random Vertical and Horizontal Shift","execution_count":null},{"metadata":{"_cell_guid":"f1397122-d153-454e-b2c3-fad53971736b","_uuid":"533c2041e8a0abe78c511e87010b4ab0808203de","trusted":true},"cell_type":"code","source":"class RandomShift(object):\n    def __init__(self, shift):\n        self.shift = shift\n        \n    @staticmethod\n    def get_params(shift):\n        hshift, vshift = np.random.uniform(-shift, shift, size=2)\n        return hshift, vshift \n    \n    def __call__(self, img):\n        hshift, vshift = self.get_params(self.shift)        \n        return img.transform(img.size, Image.AFFINE, (1,0,hshift,0,1,vshift), resample=Image.BICUBIC, fill=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b23f5c0f-045a-4beb-afc1-ae8b696f9c65","_uuid":"3d697f1ef60768dd886819fc9bb77d504086cbac"},"cell_type":"markdown","source":"## Load the Data into Tensors\nFor the training set, apply random rotation within the range of (-45, 45) degrees, shift by (-3, 3) pixels\nand normalize pixel values to [-1, 1].  For the test set, only apply nomalization.","execution_count":null},{"metadata":{"_cell_guid":"d0e41412-1be1-4ca6-a5f8-d7839d2e7010","_uuid":"5f6e6b26a71c7ca9fc3126265ad8d27491a849f5","trusted":true},"cell_type":"code","source":"train_dataset = MNIST_data(INPUT_DIR + '/train.csv', transform=transforms.Compose(\n                            [transforms.ToPILImage(), RandomRotation(degrees=20), RandomShift(3),\n                             transforms.ToTensor(), transforms.Normalize(mean=(0.5,), std=(0.5,))]))\ntest_dataset = MNIST_data(INPUT_DIR + '/test.csv')\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"351ce32d-7f9d-4ea9-983a-bf99aeaf63d8","_uuid":"faaa2bd7a8b30fbf925b34fa6dd1eca9655a756e"},"cell_type":"markdown","source":"## Network Structure","execution_count":null},{"metadata":{"_cell_guid":"766abaee-5811-4963-a4ac-69d910d9eead","_uuid":"875f53c04b120b40cd695c57a774165259183413","trusted":true},"cell_type":"code","source":"class Net(nn.Module):    \n    def __init__(self):\n        super(Net, self).__init__()\n          \n        self.features = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(32, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(64, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n          \n        self.classifier = nn.Sequential(\n            nn.Dropout(p = 0.5),\n            nn.Linear(64 * 7 * 7, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p = 0.5),\n            nn.Linear(512, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p = 0.5),\n            nn.Linear(512, 10),\n        )\n          \n        for m in self.features.children():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n        \n        for m in self.classifier.children():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n            elif isinstance(m, nn.BatchNorm1d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5b36bb2e-f4ce-4831-b829-0b737b88c014","_uuid":"3227d3f557f1770c43721ea7fc9bfc11d313e177","trusted":true},"cell_type":"code","source":"model = Net()\n\noptimizer = optim.Adam(model.parameters(), lr=0.003)\ncriterion = nn.CrossEntropyLoss()\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    criterion = criterion.cuda()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"41fecd4e-d764-4da2-8efd-e94b5b243770","_uuid":"f1ad41846a5182bacb284b362bac52d0c17bd29e"},"cell_type":"markdown","source":"## Training and Evaluation","execution_count":null},{"metadata":{"_cell_guid":"be618460-65ae-4dfc-b106-1970019b984f","_uuid":"397c6aa3d575226f3fb0c8a3a8a30889bcd8e255","trusted":true},"cell_type":"code","source":"def train(epoch):\n    model.train()\n    optimizer.step()\n    exp_lr_scheduler.step()\n\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = Variable(data), Variable(target)\n        \n        if torch.cuda.is_available():\n            data = data.cuda()\n            target = target.cuda()\n        \n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        \n        if (batch_idx + 1)% 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset), 100. * (batch_idx + 1) / len(train_loader), loss.item()))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3901fd52-c016-4b7f-bcb4-80a2449b43f4","_uuid":"977bf7d8ca46333bb53b7b44be40942d0e82572a","trusted":true},"cell_type":"code","source":"def evaluate(data_loader):\n    model.eval()\n    loss = 0\n    correct = 0\n    \n    with torch.no_grad():\n        for data, target in data_loader:\n            data, target = Variable(data), Variable(target)\n            if torch.cuda.is_available():\n                data = data.cuda()\n                target = target.cuda()\n\n            output = model(data)\n            loss += F.cross_entropy(output, target,  reduction='sum').item()\n            pred = output.data.max(1, keepdim=True)[1]\n            correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n        \n    loss /= len(data_loader.dataset)\n        \n    print('\\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(data_loader.dataset), 100. * correct / len(data_loader.dataset)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1edb9d12-9411-4fc1-8a3e-045713b6b448","_uuid":"a4c4986684b3915e329a6fc5ab380a58cb07bec7"},"cell_type":"markdown","source":"### Train the network\n\nReaches 0.995 accuracy on test set after 50 epochs","execution_count":null},{"metadata":{"_cell_guid":"c7b3146c-0beb-438d-a8de-15f6afa11ba3","_uuid":"6ca8fe3ded9fcd5dc09350aff412d878eac0df6a","scrolled":false,"trusted":true},"cell_type":"code","source":"for epoch in range(N_EPOCHS):\n    train(epoch)\n    evaluate(train_loader)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3357ea6a-db51-4be7-b101-933720ad5fca","_uuid":"f043414ac704967a9ab2b659fe5567a66796d6ba"},"cell_type":"markdown","source":"## Prediction on Test Set","execution_count":null},{"metadata":{"_cell_guid":"e00a390b-0aaf-4119-926b-03ffe486e99d","_uuid":"9fdbe41834761b6b91cfe4eebdfc88ce29f0723e","trusted":true},"cell_type":"code","source":"def prediciton(data_loader):\n    model.eval()\n    test_pred = torch.LongTensor()\n    \n    with torch.no_grad():\n        for i, data in enumerate(data_loader):\n            data = Variable(data)\n            if torch.cuda.is_available():\n                data = data.cuda()\n\n            output = model(data)\n            pred = output.cpu().data.max(1, keepdim=True)[1]\n            test_pred = torch.cat((test_pred, pred), dim=0)\n        \n    return test_pred","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c6cc74dc-8de0-403b-abed-3ead97f0348c","_uuid":"fe47f6c8c7bdfe85779ebfcf7a74e361f3b7c8a0","trusted":true},"cell_type":"code","source":"test_pred = prediciton(test_loader)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6de7f1ff-4c33-4b1e-9316-ac022ad0287d","_uuid":"4ef57831bbeeb792e7d3a847198f002ebe02f8a5","trusted":true},"cell_type":"code","source":"out_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy()], columns=['ImageId', 'Label'])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b9422a68-66e5-4bc1-af9f-1d7d02576a34","_uuid":"899374536c219a3b6208a4947046b6a46515f24b","scrolled":true,"trusted":true},"cell_type":"code","source":"out_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9f53df55-4569-4f11-a038-f54a1e200161","_uuid":"42b9c61e3462f471d9cae883880232882de0f21f","trusted":true},"cell_type":"code","source":"out_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}