{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook trains a cnn model on Kannada digits recognition with <b>pytorch</b>, <b>fastai</b>, and faster image transformation package <b>albumentations</b> rather than torchvision."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"import os, time, pickle, random\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import LambdaLR, StepLR, MultiStepLR, ReduceLROnPlateau\nfrom torch.utils.data import Dataset, DataLoader\n\nimport fastai\nfrom fastai.train import Learner\nfrom fastai.train import DataBunch\nfrom fastai.metrics import accuracy\nfrom fastai.callbacks import *\nfrom fastai.basic_data import DatasetType\n\nfrom PIL import Image\nfrom torchvision import models\nimport torchvision.datasets as dset\nimport torchvision.transforms as T\n\nimport albumentations as albu\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ndtype = torch.float32\n\nimg_size = 28\nbs = 256\nlr = 5e-2\nepochs = 28","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/Kannada-MNIST/train.csv')\ndev = pd.read_csv('../input/Kannada-MNIST/Dig-MNIST.csv')\ntest = pd.read_csv('../input/Kannada-MNIST/test.csv')\n\ntrain.shape, dev.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x, train_y = train.values[:,1:], train.values[:,0]\nx_dev, y_dev = dev.values[:,1:], dev.values[:,0]\nx_test = test.values[:,1:]\n\ndel train, dev, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print((train_x.mean(), x_dev.mean(), x_test.mean()))\nx_mean, x_std = np.concatenate([train_x, x_dev, x_test], 0).mean() / 255, np.concatenate([train_x, x_dev, x_test], 0).std() / 255\nprint(x_mean, x_std)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Observations from images and model results show that data distribution is different in 3 given datasets train/test/Dig-MNIST."},{"metadata":{"trusted":true},"cell_type":"code","source":"# train images\nfig, axes = plt.subplots(10, 10, figsize=(10,10))\n\nfor i in range(10):\n    ims = train_x[train_y == i]\n    axes[0][i].set_title(i)\n    for j in range(10):\n        axes[j][i].axis('off')\n        axes[j][i].imshow(ims[j,:].reshape(28, 28), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dig-MNIST images\nfig, axes = plt.subplots(10, 10, figsize=(10,10))\n\nfor i in range(10):\n    ims = x_dev[y_dev == i]\n    axes[0][i].set_title(i)\n    for j in range(10):\n        axes[j][i].axis('off')\n        axes[j][i].imshow(ims[j,:].reshape(28, 28), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test images\nfig, axes = plt.subplots(5, 5, figsize=(8,8))\n\nims = x_test[np.random.choice(len(x_test), 25),:]\nfor i, im in enumerate(ims):\n    ax = axes[i//5, i%5]\n    ax.axis('off')\n    ax.imshow(im.reshape(28,28), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"norm = {'mean': x_mean, 'std': x_std}\n\ntsfm_aug = albu.Compose([\n                         albu.Resize(img_size, img_size),\n                         albu.RandomContrast(limit=0.5),\n                         albu.RandomBrightness(limit=0.5),\n                         albu.ShiftScaleRotate(scale_limit=0.2, rotate_limit=15, shift_limit=0.25, border_mode=0),\n            ])\n\ntsfm_normal = albu.Compose([\n                            albu.Resize(img_size, img_size),\n                            \n            ])\n\nclass Kannada_mnist(Dataset):\n    def __init__(self, x, y=None, transforms=tsfm_normal, label_smooth=0):\n        assert 0. <= label_smooth <= 1.\n        self.x = x\n        self.y = y\n        self.transforms = transforms\n        self.label_smooth = label_smooth\n\n    def __getitem__(self, idx):\n        img = self.transforms(image=self.x[idx].astype('uint8'))['image'] / 255.\n        img = (img - norm['mean']) / norm['std']\n        img = img[None,:].astype('float32')\n        if self.y is None:\n            return img\n        if self.label_smooth > 0:\n            label = np.zeros(10)\n            label[self.y[idx]] += 1\n            label = label*(1-self.label_smooth) + self.label_smooth / 10\n        else:\n            label = self.y[idx]\n        return img, label\n\n    def __len__(self):\n        return len(self.x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Flatten(nn.Module):\n    def forward(self, x): return x.view(x.size(0), -1)\n\nclass AdaptiveConcatPool2d(nn.Module):\n    \"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`,\"\n    \"a module from fastai v1.\"\n    def __init__(self, output_size=None):\n        \"Output will be 2*output_size or 2 if output_size is None\"\n        super().__init__()\n        self.output_size = output_size or 1\n        self.ap = nn.AdaptiveAvgPool2d(self.output_size)\n        self.mp = nn.AdaptiveMaxPool2d(self.output_size)\n    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)\n\ndef vgg_style():\n    return nn.Sequential(\n        nn.Conv2d(1, 64, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(64),\n        nn.Conv2d(64, 64, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(64),\n        nn.MaxPool2d(2, stride=2),\n        nn.Conv2d(64, 128, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(128),\n        nn.Conv2d(128, 128, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(128),\n        nn.MaxPool2d(2, stride=2),\n        nn.Conv2d(128, 256, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(256),\n        nn.Conv2d(256, 256, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(256),\n        nn.Conv2d(256, 256, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(256),\n        nn.Conv2d(256, 256, 3, padding=1),\n        nn.PReLU(),\n        nn.BatchNorm2d(256),\n        AdaptiveConcatPool2d((3, 3)),\n        Flatten(),\n        nn.Linear(256 * 9 * 2, 256),\n        nn.ReLU(inplace=True),\n        nn.BatchNorm1d(256),\n        nn.Dropout(p=0.2),\n        nn.Linear(256, 10),\n    )\n\ndef model_test():\n    x = torch.zeros((64,1,28,28), dtype=dtype)\n    model = vgg_style()\n    model = model.to(device)\n    print(model(x.to(device)).size())\n\nmodel_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the cnn"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_torch(seed=0):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\ndef get_preds(test_loader, model, dset_type='test'):\n    model.eval()\n    scores = []\n    with torch.no_grad():\n        for x in test_loader:\n            if dset_type == 'val': x = x[0]\n            x = x.to(device=device, dtype=dtype)\n            score = model(x)\n            scores.append(F.softmax(score, -1).cpu().numpy())\n    scores = np.concatenate(scores)\n    return scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(train_x, train_y, test_size=0.2, random_state=0)\n\ntrain_dataset = Kannada_mnist(x_train.reshape(-1, 28, 28), y_train, transforms=tsfm_aug)\nval_dataset = Kannada_mnist(x_val.reshape(-1, 28, 28), y_val)\ndev_dataset = Kannada_mnist(x_dev.reshape(-1, 28, 28), y_dev)\ntest_dataset = Kannada_mnist(x_test.reshape(-1, 28, 28))\n\ntrain_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=bs, shuffle=False)\ndev_loader = DataLoader(dev_dataset, batch_size=bs, shuffle=False)\ntest_loader = DataLoader(test_dataset, batch_size=bs, shuffle=False)\n\ndatabunch = DataBunch(train_dl=train_loader, valid_dl=val_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_torch()\nmodel = vgg_style()\nlearn = Learner(databunch, model, loss_func=F.cross_entropy, metrics=accuracy)\nlearn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(epochs, max_lr=lr)\n\nprint()\nprint('GPU memory:')\nprint(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\nprint(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dev_preds = get_preds(dev_loader, model, 'val')\ntest_preds = get_preds(test_loader, model)\ndev_preds = np.argmax(dev_preds, -1)\ntest_preds = np.argmax(test_preds, -1)\nprint('Dev. ACC: %.4f' %((dev_preds == dev_dataset.y).mean(),))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id': np.arange(len(test_preds)), 'label': test_preds})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Comparison of predictions and true images"},{"metadata":{"trusted":true},"cell_type":"code","source":"dev_preds[:9]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(3, 3, figsize=(8,8))\n\nims = x_dev[:9,:]\nfor i, im in enumerate(ims):\n    ax = axes[i//3,i%3]\n    ax.imshow(im.reshape(28,28), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds[:9]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(3, 3, figsize=(8,8))\n\nims = x_test[:9,:]\nfor i, im in enumerate(ims):\n    ax = axes[i//3,i%3]\n    ax.imshow(im.reshape(28,28), cmap='gray')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}