{"cells":[{"metadata":{"id":"H3Gj8PYJLfcP","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import transforms\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\nplt.ion()","execution_count":null,"outputs":[]},{"metadata":{"id":"aPLkJzIwZhmr","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"dataset_path = '/kaggle/input/Kannada-MNIST/'\noutput_path = '/kaggle/working/'","execution_count":null,"outputs":[]},{"metadata":{"id":"0sxFKA8DLfcc","colab_type":"code","outputId":"315edaca-0587-4b2f-a48d-15a99ac3eb80","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Using %r' % device)\nbatch_size = 1024\nnum_workers = 4\nnum_folds = 4\nnum_epochs = 50","execution_count":null,"outputs":[]},{"metadata":{"id":"4iZJ44dNgAkv","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"csv_cache = {}\ndef read_csv(path):\n    if path in csv_cache:\n        return csv_cache[path]\n    else:\n        frame = pd.read_csv(path)\n        csv_cache[path] = frame\n        return frame\n\nclass MNIST(torch.utils.data.Dataset):\n\n    def __init__(self, *paths, train=True, transform=None, split=None):\n        self.train = train\n        self.transform = transform\n        values = pd.concat(read_csv(path) for path in paths).values\n        if train:\n            self.labels = values[:, 0].astype('int64')\n            self.images = values[:, 1:].astype('float32').reshape(-1, 28, 28)\n        else:\n            # First column dropped since it's a 0 to 5000 index\n            self.images = values.astype('float32')[:, 1:].reshape(-1, 28, 28)\n\n        if split is not None:\n            if train:\n                self.labels = self.labels[split]\n            self.images = self.images[split]\n\n    def __len__(self):\n        return self.images.shape[0]\n\n    def __getitem__(self, key):\n        if torch.is_tensor(key):\n            key = tuple(key.tolist())\n        if isinstance(key, tuple):\n            raise NotImplementedError\n        elif isinstance(key, slice):\n            image = self.images[key, :, :].copy()\n            if self.transform is not None:\n                for i in range(image.shape[0]):\n                    image[i] = self.transform(image[i])\n        elif isinstance(key, int):\n            image = self.images[key]\n            if self.transform is not None:\n                image = self.transform(image)\n\n        if self.train:\n            label = self.labels[key]\n            return image, label\n        else:\n            return image","execution_count":null,"outputs":[]},{"metadata":{"id":"srKoPvPTLfcm","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"augmented_transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomAffine(degrees=10, translate=(0.25, 0.25),\n                            scale=(0.9, 1.1), shear=10,\n                            fillcolor=0),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(128,), std=(128,)),\n])\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(128,), std=(128,)),\n])\n\ntrainsets = []\ntrainloaders = []\nfor i in range(num_folds):\n    trainset = MNIST(os.path.join(dataset_path, 'train.csv'), train=True,\n                     transform=augmented_transform,\n                     split=np.arange(i * 60000 // num_folds, (i + 1) * 60000 // num_folds))\n    trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n                                              shuffle=True, num_workers=num_workers,\n                                              pin_memory=True)\n    trainsets.append(trainset)\n    trainloaders.append(trainloader)\n\ntestset = MNIST(os.path.join(dataset_path, 'test.csv'), train=False,\n                transform=transform)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=4,\n                                         shuffle=False, num_workers=num_workers,\n                                         pin_memory=True)\n\n# Seems like a discrepency between the training set & Dig-MNIST\n# Just checked the docs it's since Dig-MNIST is not sampled the same way\n# So we'll use it as a harsh validation set\nvalset = MNIST(os.path.join(dataset_path, 'Dig-MNIST.csv'), train=True,\n               transform=transform)\nvalloader = torch.utils.data.DataLoader(valset, batch_size=batch_size,\n                                        shuffle=False, num_workers=num_workers,\n                                        pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"hQbI_H7vLfcu","colab_type":"code","outputId":"c658f7eb-44ab-4f8d-9364-d59eff9be4b1","colab":{"base_uri":"https://localhost:8080/","height":301},"trusted":true},"cell_type":"code","source":"dataiter = iter(trainloaders[0])\nimages, labels = dataiter.next()\nimages = images[:64, ...]\nlabels = labels[:64]\nplt.figure(figsize=(20, 20))\nplt.imshow(torchvision.utils.make_grid(images).numpy().transpose((1, 2, 0)) / 2 + 0.5,\n           cmap='gray')\nplt.axis('off')\nplt.title(', '.join(str(label) for label in labels.tolist()));","execution_count":null,"outputs":[]},{"metadata":{"id":"6Cu-Tk5iLfc3","colab_type":"code","outputId":"41fdb530-ac72-45a4-9c00-b9f685590668","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self):\n        super(Model, self).__init__()\n        self.conv1 = nn.Sequential(\n#             nn.Conv2d(1, 64, kernel_size=5, stride=1, padding=2),\n#             nn.BatchNorm2d(64),\n#             nn.ReLU(),\n#             nn.Conv2d(64, 64, kernel_size=5, stride=1, padding=2),\n#             nn.BatchNorm2d(64),\n#             nn.ReLU(),\n#             nn.MaxPool2d(kernel_size=2, stride=2),\n#             nn.Dropout(0.4),\n\n#             nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(64),\n#             nn.ReLU(),\n#             nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(64),\n#             nn.ReLU(),\n#             nn.Dropout(0.4),\n\n#             nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(128),\n#             nn.ReLU(),\n#             nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(128),\n#             nn.ReLU(),\n#             nn.MaxPool2d(kernel_size=2, stride=2),\n#             nn.Dropout(0.4),\n\n#             nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(256),\n#             nn.ReLU(),\n#             nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(256),\n#             nn.ReLU(),\n#             nn.Dropout(0.4),\n\n            nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(64, eps=1e-5, momentum=0.1),\n            nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(64, eps=1e-5, momentum=0.1),\n            nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(64, eps=1e-5, momentum=0.1),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Dropout(0.2),\n\n            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(128, eps=1e-5, momentum=0.1),\n            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(128, eps=1e-5, momentum=0.1),\n            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(128, eps=1e-5, momentum=0.1),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Dropout(0.2),\n\n            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(256, eps=1e-5, momentum=0.1),\n            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm2d(256, eps=1e-5, momentum=0.1),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Dropout(0.2),\n        )\n        self.dense = nn.Sequential(\n#             nn.Linear(7 * 7 * 256, 256),\n#             nn.BatchNorm1d(256),\n#             nn.ReLU(),\n#             nn.Dropout(0.4),\n#             nn.Linear(256, 128),\n#             nn.BatchNorm1d(128),\n#             nn.ReLU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(128, 10),\n\n            nn.Linear(2304, 256),\n            nn.LeakyReLU(0.1),\n            nn.BatchNorm1d(256),\n            nn.Linear(256, 10),\n        )\n    def forward(self, x):\n        x = self.conv1(x)\n        x = x.flatten(start_dim=1)\n        x = self.dense(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\nfor _ in range(num_folds):\n    model = Model().to(device)\n    print(model(iter(trainloader).next()[0].to(device)).argmax(1).tolist())\n    models.append(model)\nmodels","execution_count":null,"outputs":[]},{"metadata":{"id":"PSRJdSFGLfc-","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizers = [torch.optim.RMSprop(model.parameters(), lr=0.002,\n                                  alpha=0.9, momentum=0.1,\n                                  eps=1e-7, centered=True)\n              for model in models]\nschedulers = [torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',\n                                                         factor=0.25, patience=2,\n                                                         verbose=True, min_lr=0.00001)\n              for optimizer in optimizers]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"id":"VA_MZmiXLfdE","colab_type":"code","outputId":"aafb58d6-9a23-4cb0-bbcf-b896084e524f","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":true},"cell_type":"code","source":"histories = []\nfor k, (optimizer, model) in enumerate(zip(optimizers, models)):\n    schedulers[k]\n    histories.append([])\n    if k > 0:\n        print()\n        print()\n        print()\n    print(f'Model fold {k + 1}')\n    trainsets[k].transform = transform\n    for epoch in range(num_epochs):\n        sample = 0\n        running_total = running_errors = running_loss = 0\n        epoch_total = epoch_errors = epoch_loss = 0\n        model.train()\n        for i, trainloader in enumerate(trainloaders):\n            if i == k:\n                continue\n            for images, labels in trainloader:\n                optimizer.zero_grad()\n                outputs = model(images.to(device))\n                loss = criterion(outputs, labels.to(device))\n                loss.backward()\n                optimizer.step()\n\n                total = labels.size(0)\n                errors = (outputs.argmax(1).cpu() != labels).sum().item()\n                loss = loss.item()\n                running_total += total\n                running_errors += errors\n                running_loss += loss\n                epoch_total += total\n                epoch_errors += errors\n                epoch_loss += loss\n\n                if sample % 10 == 9:\n                    print(f'\\rEpoch {epoch + 1}, sample {(sample + 1) * batch_size:6d}, '\n                          f'loss {running_loss / running_total:.5f}, '\n                          f'acc {(1 - running_errors / running_total) * 100:3.2f}',\n                          end='')\n                    running_total = running_errors = running_loss = 0\n                sample += 1\n\n        model.eval()\n        total = 0\n        corrects = 0\n        for i, (images, labels) in enumerate(trainloaders[k]):\n#         for i, (images, labels) in enumerate(valloader):\n            total += labels.size(0)\n            corrects += (model(images.to(device)).argmax(1) == labels.to(device)).sum().cpu().item()\n            print(f'\\r{total} / {len(valset)} inferred', end='')\n        print('\\r', end='')\n        print(f'Epoch {epoch + 1}                                         ')\n        print(f'- Val acc: {100 * corrects / total:3.2f}%')\n        print(f'- Train acc: {100 * (1 - epoch_errors / epoch_total):3.2f}%, '\n              f'loss: {epoch_loss:.5f}')\n        histories[k].append((corrects / total, 1 - epoch_errors / epoch_total))\n        schedulers[k].step(epoch_loss)\n    trainsets[k].transform = augmented_transform","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for history in histories:\n    plt.plot(range(1, len(history) + 1), list(zip(*history))[0], label='Validation accuracy')\n    plt.plot(range(1, len(history) + 1), list(zip(*history))[1], label='Training accuracy')\n    plt.legend()\n    plt.ylim(None, 1)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"GoX0mYR8LfdI","colab_type":"code","outputId":"e8211597-2603-4529-9812-999093729716","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"for model in models:\n    model.eval()\ntotal = 0\ncorrects = 0\npredictions = []\nfor i, (images, labels) in enumerate(valloader):\n    total += labels.size(0)\n    prediction = sum(model(images.to(device)) for model in models).argmax(1)\n    corrects += (prediction == labels.to(device)).sum().cpu().item()\n    predictions.extend(prediction.tolist())\n    print(f'\\r{total} / {len(valset)} inferred', end='')\nprint('\\r', end='')\nprint(f'{100 * corrects / total:.2f}%                  ')\nconf = confusion_matrix(valset[:][1], predictions)\nconf = pd.DataFrame(conf, index=range(0,10), columns=range(0,10))\nplt.figure(figsize=(12,10))\nsns.heatmap(conf, cmap=\"coolwarm\", annot=True , fmt=\"d\");","execution_count":null,"outputs":[]},{"metadata":{"id":"zogLiEsWLfdP","colab_type":"code","outputId":"56be7eaa-522c-47c0-9ee3-1f8b1581dbe1","colab":{"base_uri":"https://localhost:8080/","height":125},"trusted":true},"cell_type":"code","source":"dataiter = iter(valloader)\nimages, labels = dataiter.next()\nimages = images[:8, :, :, :]\nlabels = labels[:8]\nplt.imshow(torchvision.utils.make_grid(images).numpy().transpose((1, 2, 0)) / 2 + 0.5,\n           cmap='gray')\nplt.axis('off')\nplt.title(', '.join(str(label) for label in labels.tolist()))\npredictions = sum(model(images.to(device)) for model in models)\npreds = ', '.join(str(prediction.item()) for prediction in predictions.argmax(1).cpu())\nprint(f'Predictions:  {preds}')\nprint(f'Ground truth: {\", \".join(str(label) for label in labels.tolist())}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, model in enumerate(models):\n    torch.save(model.state_dict(), os.path.join(output_path, f'model-{i + 1}.pt'))","execution_count":null,"outputs":[]},{"metadata":{"id":"QmCBGrCczn9s","colab_type":"code","outputId":"69c8a68a-ab42-42fe-d24e-d603a44bc4f6","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"submission = []\nfor images in testloader:\n    predictions = sum(model(images.to(device)) for model in models)\n    submission.extend(predictions.argmax(1).tolist())\nlen(submission)","execution_count":null,"outputs":[]},{"metadata":{"id":"i_B_rm_NLfd2","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"df = pd.DataFrame.from_records(np.array(submission).reshape(-1, 1))\ndf.to_csv(os.path.join(output_path, 'submission.csv'),\n          index_label='id', header=['label'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"},"colab":{"name":"playground.ipynb","provenance":[],"collapsed_sections":[]},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}