{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nprint(train.head())\nprint(test.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MNISTDataset(Dataset):\n    \n    def __init__(self, file_path, transform=None):\n        self.data = pd.read_csv(file_path)\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        image = self.data.iloc[index, 1:].values.astype(np.uint8).reshape((28, 28, 1))\n        label = self.data.iloc[index, 0]\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vis_transform = transforms.Compose([\n    transforms.ToPILImage(), # because the input dtype is numpy.ndarray\n    transforms.RandomRotation(10),\n    transforms.RandomResizedCrop(28, scale=(0.9,1.1)),\n    transforms.ToTensor(), \n])\n\nvis_dataset = MNISTDataset(\"/kaggle/input/digit-recognizer/train.csv\", transform=vis_transform)\nvis_loader = DataLoader(vis_dataset, batch_size=8, shuffle=True)\nimages, labels = iter(vis_loader).next()\n\ngrid = torchvision.utils.make_grid(images)\n\nplt.imshow(grid.numpy().transpose((1, 2, 0)))\nplt.axis('off')\nplt.title(labels.numpy());","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LeNet(nn.Module):\n\n    def __init__(self):\n        super(LeNet, self).__init__()\n        # 1 input image channel, 6 output channels, 5x5 square convolution\n        # kernel\n        self.conv1 = nn.Conv2d(1, 6, 5)\n        self.bn1 = nn.BatchNorm2d(6)\n        self.conv2 = nn.Conv2d(6, 16, 5)\n        self.bn2 = nn.BatchNorm2d(16)\n        # an affine operation: y = Wx + b\n        self.fc1 = nn.Linear(16 * 4 * 4, 120)\n        self.bn3 = nn.BatchNorm1d(120)\n        self.fc2 = nn.Linear(120, 84)\n        self.bn4 = nn.BatchNorm1d(84)\n        self.fc3 = nn.Linear(84, 10)\n\n    def forward(self, x):\n        # Max pooling over a (2, 2) window\n        x = F.max_pool2d(F.relu(self.bn1(self.conv1(x))), (2, 2))\n        # If the size is a square you can only specify a single number\n        x = F.max_pool2d(F.relu(self.bn2(self.conv2(x))), 2)\n        x = x.view(-1, 16 * 4 * 4)\n        x = F.relu(self.bn3(self.fc1(x)))\n        x = F.relu(self.bn4(self.fc2(x)))\n        x = self.fc3(x)\n        return x\n\nnet = LeNet()\nprint(net)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(net.parameters(), lr=0.005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport copy\n\ndef train_model(model, dataloaders, criterion, optimizer, num_epochs=5):\n    \"\"\" From tutorial: https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html#model-training-and-validation-code\"\"\"\n    since = time.time()\n\n    val_acc_history = []\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    # Get model outputs and calculate loss\n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                    _, preds = torch.max(outputs, 1)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pick train-validate split\nfrom torch.utils.data import Subset\n\ntrain_transform = transforms.Compose([\n    transforms.ToPILImage(), # because the input dtype is numpy.ndarray\n    transforms.RandomRotation(10),\n    transforms.RandomResizedCrop(28, scale=(0.9,1.1)),\n    transforms.ToTensor(), \n])\n\ntrain_dataset = MNISTDataset(\"/kaggle/input/digit-recognizer/train.csv\", transform=train_transform)\nvalid_dataset = MNISTDataset(\"/kaggle/input/digit-recognizer/train.csv\", transform=torchvision.transforms.ToTensor())\n\ndef get_train_valid_loader(train_dataset, valid_dataset, batch_size=64, random_seed=129, shuffle=True, valid_size=0.1):\n    num_train = len(train_dataset)\n    indices = list(range(num_train))\n    split = int(np.floor(valid_size * num_train))\n\n    np.random.seed(random_seed)\n    np.random.shuffle(indices)\n\n    train_idx, valid_idx = indices[split:], indices[:split]\n    \n    train_loader = torch.utils.data.DataLoader(Subset(train_dataset, train_idx), batch_size=batch_size)\n    valid_loader = torch.utils.data.DataLoader(Subset(valid_dataset, valid_idx), batch_size=batch_size)\n\n    return (train_loader, valid_loader)\n\ntrain_loader, valid_loader = get_train_valid_loader(train_dataset, valid_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_model(net, {'train': train_loader, 'val': valid_loader}, criterion, optimizer, num_epochs=40)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MNISTTest(Dataset):\n    \n    def __init__(self, file_path, transform=None):\n        self.data = pd.read_csv(file_path)\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        image = self.data.iloc[index, :].values.astype(np.uint8).reshape((28, 28, 1))\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = MNISTTest(\"/kaggle/input/digit-recognizer/test.csv\", transform=torchvision.transforms.ToTensor())\ntest_loader = DataLoader(test_dataset, batch_size=8, shuffle=False)\nimages = iter(test_loader).next()\nlabels = torch.max(net(images).data, 1).indices\n\ngrid = torchvision.utils.make_grid(images)\n\nplt.imshow(grid.numpy().transpose((1, 2, 0)))\nplt.axis('off')\nplt.title(labels.numpy());","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\nwith torch.no_grad():\n    for images in test_loader:\n        labels = torch.max(net(images).data, 1).indices\n        results.extend(labels.numpy().tolist())\nresults = np.array(results)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"MNIST-CNN.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}