{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\n\nimport torch\nimport torch.nn.functional as func\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\n\nimport torchvision\nfrom torchvision import transforms\n\nimport tqdm\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n%matplotlib inline\n\nimport warnings\nfrom IPython.display import clear_output\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"torch.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class CapsuleLayer(nn.Module):\n    def __init__(self, num_caps, num_routes, in_channels, out_channels, k_size=None, \n                 stride=None, num_rounds=3,\n                 use_padding=False):\n        super(CapsuleLayer, self).__init__()\n\n        self.num_routes = num_routes\n        self.num_rounds = num_rounds\n        self.num_caps = num_caps\n\n        if num_routes != -1:\n            self.W = nn.Parameter(torch.randn(num_caps, num_routes, in_channels, out_channels))\n        else:\n            self.capsules = nn.ModuleList(\n                [nn.Conv2d(in_channels, out_channels, kernel_size=k_size, stride=stride,\n                           padding=(k_size-1)//2 if use_padding else 0)\n                 for _ in range(num_caps)]\n            )\n\n    @staticmethod\n    def squash(x, dim=-1):\n        s_norm = (x**2).sum(dim=dim, keepdim=True)\n        scaled = s_norm / (1 + s_norm)\n        return scaled * x / torch.sqrt(s_norm)\n\n    def forward(self, x):\n\n        if self.num_routes != -1:\n            priors = x[None, :, :, None, :] @ self.W[:, None, :, :, :]\n            logits = torch.zeros(*priors.size(), device=torch.device('cuda' if torch.cuda.is_available() else 'cpu'),\n                                 requires_grad = True)\n            for i in range(self.num_rounds):\n                probs = func.softmax(logits, dim=2)\n                outps = self.squash((probs * priors).sum(dim=2, keepdim=True))\n\n                if i != self.num_rounds - 1:\n                    del_logits = (priors * outps).sum(dim=-1, keepdim=True)\n                    logits = logits + del_logits\n        else:\n            batch_size = x.size(0)\n            outps = [cap(x).view(batch_size, -1, 1) for cap in self.capsules]\n            outps = torch.cat(outps, dim=-1)\n            outps = self.squash(outps)\n        return outps\n\ndef conv_size(shape, k = 9, s = 1, p = False):\n    H, W = shape\n    if p:\n        pad = (k-1)//2\n    else:\n        pad = 0\n\n    Ho = math.floor(((H + 2*pad - (k - 1) - 1)/s) + 1)\n    Wo = math.floor(((W + 2*pad - (k - 1) - 1)/s) + 1)\n\n    return Ho, Wo\n\nclass CapsuleNetwork(nn.Module):\n    def __init__(self, img_size, ic_channels, num_pcaps, num_classes, num_coc, num_doc, mode='mono', use_padding=False):\n        super(CapsuleNetwork, self).__init__()\n\n        self.initial_conv = nn.Conv2d(in_channels=1 if mode=='mono' else 3, out_channels=ic_channels, kernel_size=9, stride=1)\n        Ho, Wo = conv_size(img_size, k=9, s=1, p=False)\n\n        self.p_caps = CapsuleLayer(num_caps=num_pcaps, num_routes=-1, in_channels=ic_channels, out_channels=num_coc,\n                                   k_size=9, stride=2)\n        Ho, Wo = conv_size((Ho, Wo), k=9, s=2, p=use_padding)\n\n        self.d_caps = CapsuleLayer(num_caps=num_classes, num_routes=num_coc*Ho*Wo, in_channels=num_pcaps, out_channels=num_doc)\n\n        self.decoder = Decoder(num_doc)\n\n    def forward(self, x):\n        x = func.relu(self.initial_conv(x))\n        x = self.p_caps(x)\n        x = self.d_caps(x).squeeze().transpose(0,1)\n\n        classes = (x ** 2).sum(dim=-1) ** 0.5\n        classes = func.softmax(classes, dim=-1)\n\n        _, max_index = classes.max(dim=1)\n        y = torch.eye(10, device=torch.device('cuda' if torch.cuda.is_available() else 'cpu'),\n                             requires_grad = True).index_select(dim=0, index=max_index.data)\n        reconst = self.decoder((x * y[:, :, None]).view(x.size(0), -1))\n\n        return classes, reconst\n\nclass Decoder(nn.Module):\n\n    def __init__(self, in_size):\n        super(Decoder, self).__init__()\n\n        self.fc1 = nn.Linear(in_size * 10, 512)\n        self.relu = nn.ReLU()\n        self.fc2 = nn.Linear(512, 784) # 1/8\n        self.c1 = nn.Conv2d(in_channels=16, out_channels=16, kernel_size=3, padding=1) #1/4\n        self.c2 = nn.Conv2d(in_channels=16, out_channels=8, kernel_size=3, padding=1) #1/2\n        self.c3 = nn.Conv2d(in_channels=8, out_channels=1, kernel_size=3, padding=1) #1\n\n    def forward(self, x):\n        x = self.relu(self.fc1(x))\n        x = self.relu(self.fc2(x))\n        x = x.view(-1, 16, 7, 7)\n        x = self.relu(self.c1(x))\n        x = func.interpolate(x, scale_factor=2)\n        x = self.relu(self.c2(x))\n        x = func.interpolate(x, scale_factor=2)\n        x = torch.sigmoid(self.c3(x))\n        return x\n    \nclass CapsuleLoss(nn.Module):\n    def __init__(self):\n        super(CapsuleLoss, self).__init__()\n        self.reconst = nn.MSELoss(size_average=False)\n\n    def forward(self, img, label, classes, reconst):\n        label = torch.eye(10, device=torch.device('cuda' if torch.cuda.is_available() else 'cpu'),\n                  requires_grad=True).index_select(dim=0, index=label.data)\n        left = func.relu(0.9-classes) ** 2\n        right = func.relu(classes - 0.1) ** 2\n\n        margin = label * left + 0.5 * (1-label) * right\n        margin = margin.sum()\n\n        recon = self.reconst(img, reconst)\n        return (margin + 0.0005 * recon) / img.size(0)\n\n    \ndef train_capnet(name, model, train_loader, valid_loader, epochs):\n    optim = torch.optim.Adam(model.parameters(), lr=0.001)\n    dev = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    loss = CapsuleLoss()\n    \n    LR = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer=optim,\n        mode='max', \n        factor=0.1, \n        patience=3, \n        verbose=True, \n        threshold_mode='rel', \n        cooldown=2, \n        min_lr=0, \n        eps=1e-08\n    )\n    \n    model.to(dev)\n    loss.to(dev)\n    best_model = None\n    best_acc = 0.0\n    terror = []\n    tmean = []\n    acc = []\n    \n    for e in range(epochs):\n        lavg = 0\n\n        for i, data in enumerate(train_loader):\n            \n            img, label = data\n            img = (img - img.mean())/(img.std())\n            img = img.to(dev)\n            label = label.to(dev)\n            optim.zero_grad()\n            y, reconst = model(img)\n            l = loss(img, label, y, reconst)\n            lavg += l.item()\n            if i > 0:\n                lavg /= 2\n            l.backward()\n            terror.append(l.item())\n            optim.step()\n        tmean.append(sum(terror)/len(terror))\n\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for i, data in enumerate(valid_loader):\n                img, labels = data\n                img = (img - img.mean())/(img.std())\n                img = img.to(dev)\n                labels = labels.to(dev)\n                y, _ = model(img)\n                _, pred = torch.max(y.data, 1)\n                total += labels.size(0)\n                correct += (pred == labels).sum().item()\n        accuracy = correct / total\n        if accuracy > best_acc:\n            best_model = model\n            best_acc = accuracy\n        acc.append(accuracy) \n        LR.step(accuracy)\n        if e % 5 == 0:\n            print(f'model: {name}, epoch: {e}, {correct}, {total}, accuracy: {100 * (correct/total)}%')\n        \n    return best_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class MNIST(Dataset):\n    \n    def __init__(self, path, transforms = None):\n        df = pd.read_csv(path)\n        \n        if len(df.columns) == 784:\n            self.X = df.values.reshape((-1, 28, 28)).astype(np.float)[:, :, :]\n            self.y = None\n        else:\n            self.X = df.iloc[:, 1:].values.reshape((-1, 28, 28)).astype(np.float)[:, :, :]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n        \n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        X = self.X[idx]\n        X = Image.fromarray(X)\n        if self.y is not None:\n            return self.transforms(X) if self.transforms is not None else X, self.y[idx]\n        else:\n            return self.transforms(X) if self.transforms is not None else X","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"batch_size = 512\nn_caps = 8\nih, iw = 28, 28\nn_class = 10\nin_channels = 256\nprime_channels = 32\ndim_channels = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"trans = transforms.Compose([\n    transforms.RandomAffine(\n        degrees=5, \n        translate=(0.05, 0.05), \n        scale=None, \n        shear=5, \n        resample=False, \n        fillcolor=0\n    ),\n     transforms.ToTensor()\n])\ndata = MNIST(path = '../input/Kannada-MNIST/train.csv', transforms=trans)\nprint(len(data))\nvalid = MNIST(path = '../input/Kannada-MNIST/Dig-MNIST.csv', transforms=trans)\ntest = MNIST(path = '../input/Kannada-MNIST/test.csv', transforms=transforms.ToTensor())\n\ndata = torch.utils.data.ConcatDataset([data, valid])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(len(data))\nplt.imshow(data[35][0].squeeze(), 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"cap_model = [('capsule', CapsuleNetwork(\n    img_size = (ih, iw), \n    ic_channels = in_channels, \n    num_pcaps = n_caps, \n    num_classes = n_class, \n    num_coc = prime_channels, \n    num_doc = dim_channels, \n    mode='mono', \n    use_padding=False\n))] # add more models to ensemble","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_load = DataLoader(dataset=data, batch_size=batch_size, shuffle=True, num_workers=4)\nvalid_load = DataLoader(dataset=valid, batch_size=batch_size, shuffle=True, num_workers=4)\ntest_load = DataLoader(dataset=test, batch_size=batch_size, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\nbest_models = []\nprint('training using:', torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'))\nfor name, model in cap_model:    \n    best = train_capnet(name = name, model=model, train_loader=train_load, valid_loader=valid_load, epochs=30)\n    best_models.append(best)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def predicition(model, data_loader):\n    model.eval()\n    test_pred = torch.LongTensor()\n    dev = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    model.to(dev)\n    for i, d in enumerate(data_loader):\n        d, _ = d\n        d = d.to(dev)\n        d = (d-d.mean())/d.std()\n        with torch.no_grad():\n            output = model(d)\n        \n        pred = output.cpu().data.max(1, keepdim=True)[1]\n        test_pred = torch.cat((test_pred, pred), dim=0)\n        \n    return test_pred.numpy()\n\ndef predicition_cap(model, data_loader):\n    model.eval()\n    test_pred = torch.LongTensor()\n    dev = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    model.to(dev)\n    for i, d in enumerate(data_loader):\n        d, _ = d\n        d = d.to(dev)\n        d = (d-d.mean())/d.std()\n        with torch.no_grad():\n            output, _ = model(d)\n        \n        pred = output.cpu().data.max(1, keepdim=True)[1]\n        test_pred = torch.cat((test_pred, pred), dim=0)\n        \n    return test_pred.numpy()\n\ndef most_common(lst):\n    return max(set(lst), key=lst.count)\n\ndef vote_pred(models):\n    preds = []\n    for model in models:\n        pred = predicition_cap(model, test_load)\n        preds.append(pred)\n    preds = np.array(preds)\n    return preds\n    \npreds = vote_pred(best_models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pred = preds.squeeze()\nprint(pred.shape)\nif len(pred.shape) > 1:\n    pred = pred.transpose([1,0])\n    pred = np.array([most_common(list(p)) for p in pred.squeeze()])\npred = pred.reshape([-1, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = pred.astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":false},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"pycharm":{"stem_cell":{"cell_type":"raw","source":[],"metadata":{"collapsed":false}}}},"nbformat":4,"nbformat_minor":1}