{"cells":[{"metadata":{"_uuid":"40d27f5ef16aafcf0181acc3320155e595874777","_cell_guid":"18074494-9783-48af-bb66-a9d59a8f43e5"},"cell_type":"markdown","source":"# Convolutional Neural Network With PyTorch\n----------------------------------------------\n\nTraining get at 98% accuracy after about 3 epochs, Don't forget to turn GPU on train your network in a few minutes"},{"metadata":{"_uuid":"427ce49fba02f44de9e3081288fca2da38e51140","_cell_guid":"56682ac8-509e-44fb-a694-4bf1082d36d2","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.utils import make_grid\n\nimport math\nimport random\n\nimport numbers\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting Random number seeds for reproducability"},{"metadata":{"trusted":true},"cell_type":"code","source":"random_seed = 7\nrandom.seed(random_seed)\nnp.random.seed(random_seed)\ntorch.manual_seed(random_seed)\nprint('Random seed is now: {}'.format(random_seed))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a346d02febaa91c6ae6073dad4a227fe38242a9","_cell_guid":"e224069f-8b41-4390-a6d5-09ac6379982f"},"cell_type":"markdown","source":"## Explore the Data"},{"metadata":{"_uuid":"86cafb6770a21e6ed806c17c2be9897399bdf520","_cell_guid":"5b89951b-5acb-4957-b6e4-cfeb54f4a053","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\n\nn_train = len(train_df)\nn_pixels = len(train_df.columns) - 1\nn_class = len(set(train_df['label']))\n\nprint('Number of training samples: {0}'.format(n_train))\nprint('Number of training pixels: {0}'.format(n_pixels))\nprint('Number of classes: {0}'.format(n_class))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2c697381fd6210380e2350553bd43ca13194cff1","_cell_guid":"7bf732ab-a5e7-4db0-ab30-8b3f1f89bf60","trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')\n\nn_test = len(test_df)\nn_pixels = len(test_df.columns)\n\nprint('Number of train samples: {0}'.format(n_test))\nprint('Number of test pixels: {0}'.format(n_pixels))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbd7b4c21b4049d77bdd3b5d32984fdfbe7b563f","_cell_guid":"a1480b3a-5298-4193-852c-26909b3aa6ea"},"cell_type":"markdown","source":"### Display some images"},{"metadata":{"_uuid":"4cfd978d90f8fe48bf7494a2e00448c3ec48b05b","_cell_guid":"969c8fd1-72b0-46a8-af7c-62f630f626e8","trusted":true},"cell_type":"code","source":"random_sel = np.random.randint(n_train, size=8)\n\ngrid = make_grid(torch.Tensor((train_df.iloc[random_sel, 1:].as_matrix()/255.).reshape((-1, 28, 28))).unsqueeze(1), nrow=8)\nplt.rcParams['figure.figsize'] = (16, 2) # units in Inchs\nplt.imshow(grid.numpy().transpose((1,2,0)))\nplt.axis('off')\nprint(*list(train_df.iloc[random_sel, 0].values), sep = ', ')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21baf11c7663dd085150a3a21e843d67a29acb99","_cell_guid":"22f18c2b-012b-4862-91cc-7ecde9db7bfe"},"cell_type":"markdown","source":"### Histogram of the classes"},{"metadata":{"_uuid":"2aad956c1baa1e29a259902835b66b020a540c96","_cell_guid":"ecd233dc-e143-4964-87db-21b8aa8a192c","trusted":true},"cell_type":"code","source":"plt.rcParams['figure.figsize'] = (8, 5)\nplt.bar(train_df['label'].value_counts().index, train_df['label'].value_counts())\nplt.xticks(np.arange(n_class))\nplt.xlabel('Class', fontsize=16)\nplt.ylabel('Count', fontsize=16)\nplt.grid('on', axis='y')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1166411bc5c17d2147e0e29feb1df8c94b0f203a","_cell_guid":"f06156b2-2118-4a3b-a648-e15b70a5982e"},"cell_type":"markdown","source":"## Data Loader"},{"metadata":{"_uuid":"13401816f84176712146c336d11bdab8ce549bce","_cell_guid":"711ee835-cb44-4243-9f7e-60c6c4301cca","trusted":true},"cell_type":"code","source":"class MNIST_data(Dataset):\n    \"\"\"MNIST data set\"\"\"\n    \n    def __init__(self, file_path):\n        n_pixels = 28 * 28\n        df = pd.read_csv(file_path)\n        \n        if len(df.columns) == n_pixels:\n            # test data\n            self.X = df.values.reshape((-1,1,28,28)).astype(np.float) / 255\n            self.y = None\n        else:\n            # training data\n            self.X = df.iloc[:,1:].values.reshape((-1,1,28,28)).astype(np.float) / 255\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n        \n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        if self.y is not None: #in case of training and test set\n            return self.X[idx], self.y[idx] \n        else: # in case of submission datasets  \n            return self.X[idx] ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d697f1ef60768dd886819fc9bb77d504086cbac","_cell_guid":"b23f5c0f-045a-4beb-afc1-ae8b696f9c65"},"cell_type":"markdown","source":"## Load the Data into Tensors"},{"metadata":{"_uuid":"5f6e6b26a71c7ca9fc3126265ad8d27491a849f5","_cell_guid":"d0e41412-1be1-4ca6-a5f8-d7839d2e7010","trusted":true},"cell_type":"code","source":"from torch.utils.data import Subset\n\nbatch_size = 64\n\ntrain_dataset_all = MNIST_data('../input/train.csv')\n\nall_train_len = len(train_dataset_all)\nindices = list(range(all_train_len))\nvalidation_split = 0.2\nsplit = int(np.floor(validation_split * all_train_len))\nnp.random.shuffle(indices)\ntrain_indices, val_indices = indices[split:], indices[:split]\n\ntrain_dataset = Subset(train_dataset_all, train_indices)\nval_dataset = Subset(train_dataset_all, val_indices)\n\ntest_dataset = MNIST_data('../input/test.csv')\n\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                           batch_size=batch_size, shuffle=True)\nval_loader = torch.utils.data.DataLoader(dataset=val_dataset,\n                                           batch_size=batch_size, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n                                           batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"faaa2bd7a8b30fbf925b34fa6dd1eca9655a756e","_cell_guid":"351ce32d-7f9d-4ea9-983a-bf99aeaf63d8"},"cell_type":"markdown","source":"## Network Structure"},{"metadata":{"_uuid":"875f53c04b120b40cd695c57a774165259183413","_cell_guid":"766abaee-5811-4963-a4ac-69d910d9eead","trusted":true},"cell_type":"code","source":"class LeNet(nn.Module):\n    def __init__(self):\n        super(LeNet, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5)\n        self.conv2 = nn.Conv2d(6, 16, 5)\n        self.fc1   = nn.Linear(256, 120)\n        self.fc2   = nn.Linear(120, 84)\n        self.fc3   = nn.Linear(84, 10)\n\n    def forward(self, x):\n        out = F.relu(self.conv1(x))\n        out = F.max_pool2d(out, 2)\n        out = F.relu(self.conv2(out))\n        out = F.max_pool2d(out, 2)\n        out = out.view(out.size(0), -1)\n        out = F.relu(self.fc1(out))\n        out = F.relu(self.fc2(out))\n        out = self.fc3(out)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3227d3f557f1770c43721ea7fc9bfc11d313e177","_cell_guid":"5b36bb2e-f4ce-4831-b829-0b737b88c014","trusted":true},"cell_type":"code","source":"model = LeNet()\n\noptimizer = optim.Adam(model.parameters(), lr=0.003)\n\ncriterion = nn.CrossEntropyLoss()\n\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    criterion = criterion.cuda()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1ad41846a5182bacb284b362bac52d0c17bd29e","_cell_guid":"41fecd4e-d764-4da2-8efd-e94b5b243770"},"cell_type":"markdown","source":"## Training and Evaluation"},{"metadata":{"_uuid":"397c6aa3d575226f3fb0c8a3a8a30889bcd8e255","_cell_guid":"be618460-65ae-4dfc-b106-1970019b984f","trusted":true},"cell_type":"code","source":"def train(epoch):\n    model.train()\n    #exp_lr_scheduler.step()\n\n    for batch_idx, (data, target) in enumerate(train_loader):\n        \n        if torch.cuda.is_available():\n            data = data.float().cuda()\n            target = target.cuda()\n        \n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        \n        loss.backward()\n        optimizer.step()\n        \n        if (batch_idx + 1)% 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n                100. * (batch_idx + 1) / len(train_loader), loss.data.item()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"977bf7d8ca46333bb53b7b44be40942d0e82572a","_cell_guid":"3901fd52-c016-4b7f-bcb4-80a2449b43f4","trusted":true},"cell_type":"code","source":"def evaluate(data_loader):\n    model.eval()\n    loss = 0\n    correct = 0\n    \n    for data, target in data_loader:\n        if torch.cuda.is_available():\n            data = data.float().cuda()\n            target = target.cuda()\n        \n        output = model(data)\n        \n\n        pred = output.argmax(dim=1)\n        correct += (pred == target).cpu().sum()\n    loss /= len(data_loader.dataset)\n        \n    print('\\nAccuracy: {}/{} ({:.3f}%)\\n'.format(  \n         correct, len(data_loader.dataset), \n        100. * correct / len(data_loader.dataset)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4c4986684b3915e329a6fc5ab380a58cb07bec7","_cell_guid":"1edb9d12-9411-4fc1-8a3e-045713b6b448"},"cell_type":"markdown","source":"### Train the network\n\nReaches 98% accuracy on val set after about 3 epochs"},{"metadata":{"_uuid":"6ca8fe3ded9fcd5dc09350aff412d878eac0df6a","_cell_guid":"c7b3146c-0beb-438d-a8de-15f6afa11ba3","trusted":true},"cell_type":"code","source":"n_epochs = 15\n\nfor epoch in range(n_epochs):\n    train(epoch)\n    print(\"train data:\")\n    evaluate(train_loader)\n    print(\"val data:\")\n    evaluate(val_loader)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f043414ac704967a9ab2b659fe5567a66796d6ba","_cell_guid":"3357ea6a-db51-4be7-b101-933720ad5fca"},"cell_type":"markdown","source":"## Prediction on Test Set"},{"metadata":{"_uuid":"9fdbe41834761b6b91cfe4eebdfc88ce29f0723e","_cell_guid":"e00a390b-0aaf-4119-926b-03ffe486e99d","trusted":true},"cell_type":"code","source":"def prediciton(data_loader):\n    model.eval()\n    test_pred = torch.LongTensor()\n    \n    for i, data in enumerate(data_loader):\n        data = data.float()\n        if torch.cuda.is_available():\n            data = data.cuda()\n            \n        output = model(data)\n        \n        pred = output.cpu().argmax(dim=1)\n        test_pred = torch.cat((test_pred, pred), dim=0)\n        \n    return test_pred","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe47f6c8c7bdfe85779ebfcf7a74e361f3b7c8a0","_cell_guid":"c6cc74dc-8de0-403b-abed-3ead97f0348c","trusted":true},"cell_type":"code","source":"test_pred = prediciton(test_loader)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ef57831bbeeb792e7d3a847198f002ebe02f8a5","_cell_guid":"6de7f1ff-4c33-4b1e-9316-ac022ad0287d","trusted":true},"cell_type":"code","source":"out_df = pd.DataFrame({'ImageId':list(range(1, len(test_dataset) + 1)),'Label':test_pred.numpy()})","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"899374536c219a3b6208a4947046b6a46515f24b","_cell_guid":"b9422a68-66e5-4bc1-af9f-1d7d02576a34","trusted":true},"cell_type":"code","source":"out_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42b9c61e3462f471d9cae883880232882de0f21f","_cell_guid":"9f53df55-4569-4f11-a038-f54a1e200161","trusted":true},"cell_type":"code","source":"out_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","mimetype":"text/x-python","file_extension":".py","nbconvert_exporter":"python","pygments_lexer":"ipython3","codemirror_mode":{"name":"ipython","version":3},"version":"3.6.3"}},"nbformat":4,"nbformat_minor":1}