{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data as D\nimport torch.nn.functional as F\n\nimport torchvision\nfrom torchvision import transforms as T\n\nimport os, time, random, sys, math","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/Kannada-MNIST/train.csv')\ndf_test = pd.read_csv('../input/Kannada-MNIST/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Identity():\n    r\"\"\"A placeholder identity operator that is argument-insensitive.\n\n    Args:\n        args: any argument (unused)\n        kwargs: any keyword argument (unused)\n\n    \"\"\"\n    def __init__(self, *args, **kwargs):\n        pass\n    \n    def __repr__(self):\n        format_string = self.__class__.__name__ \n        return format_string\n    \n    def __call__(self, input):\n        return input\n    \nclass DataSet(D.Dataset):\n    def __init__(self, df, transform=None, mode='train'):\n        self.mode = mode\n        self.transform = transform if transform else Identity()\n        self.len = len(df)\n        self.images = df.iloc[:,1:].values.reshape(-1,28,28,1)\n        if mode == 'train':\n            self.y = df.label.values\n    \n    def __getitem__(self, index):\n        \n        image = self.images[index]\n        image = self.transform(image)\n        if self.mode == 'train':\n            return image, self.y[index]\n        else:\n            return image\n            \n    def __len__(self):\n        \"\"\"\n        Total number of samples in the dataset\n        \"\"\"\n        return self.len\n    \nclass ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=15.0, m=0.35, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        if not self.training:\n            return cosine * self.s\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=input.device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n        output *= self.s\n\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 4*1024\ndevice = 'cuda'\n\ntrfm = T.Compose([\n        T.Lambda(lambda x: x / 255),\n        T.ToTensor(),\n        T.Lambda(lambda x: x.float()),\n    ])\n\nds = DataSet(df_train[:50000], transform=trfm)\nloader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=2)\n\nvds = DataSet(df_train[50000:], transform=trfm)\nvloader = D.DataLoader(vds, batch_size=batch_size, shuffle=False, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Network(nn.Module):\n    def __init__(self):\n        super(Network, self).__init__()\n        # image starts as (1, 28, 28)\n        # Formula to compute size of image after conv/pool\n        # (size-filter+2*padding / stride) + 1\n        #                      inputs         # of filters    filter size    \n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5, stride=1, padding=2) # conv1\n        self.conv1_bn = nn.BatchNorm2d(num_features=32)\n        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2) # conv2\n        self.conv2_bn = nn.BatchNorm2d(num_features=64)\n        self.conv3 = nn.Conv2d(in_channels=64, out_channels= 128, kernel_size=3, stride=1, padding=1) # conv3\n        self.conv3_bn = nn.BatchNorm2d(num_features=128)\n        \n        self.fc1 = nn.Linear(in_features=128*6*6, out_features=1024) # linear 1\n        self.fc1_bn = nn.BatchNorm1d(num_features=1024)\n        self.fc2 = nn.Linear(in_features=1024, out_features=512) # linear 2\n        self.fc2_bn = nn.BatchNorm1d(num_features=512)\n        self.fc3 = nn.Linear(in_features=512, out_features=256) # linear 3\n        self.fc3_bn = nn.BatchNorm1d(num_features=256)\n        self.fc4 = nn.Linear(in_features=256, out_features=64) # linear 4\n        self.fc4_bn = nn.BatchNorm1d(num_features=64)\n        self.arc = ArcMarginProduct(64,10,s=17,m=0.5) # output\n    \n    def forward(self, t, y=None):\n        t = F.relu(self.conv1_bn(self.conv1(t)))\n        t = F.max_pool2d(t, kernel_size=2, stride=2) # (1, 14, 14)\n        \n        t = F.relu(self.conv2_bn(self.conv2(t)))\n        t = F.max_pool2d(t, kernel_size=2, stride=2) # (1, 7, 7)\n        \n        t = F.relu(self.conv3_bn(self.conv3(t)))\n        t = F.max_pool2d(t, kernel_size=2, stride=1) # (1, 6, 6)\n        \n        t = F.relu(self.fc1_bn(self.fc1(t.reshape(-1, 128*6*6))))\n        t = F.relu(self.fc2_bn(self.fc2(t)))\n        t = F.relu(self.fc3_bn(self.fc3(t)))\n        t = F.relu(self.fc4_bn(self.fc4(t)))\n        t = self.arc(t, y)\n        \n        return t\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Flatten(nn.Module):  \n    def forward(self, x):\n        return torch.flatten(x, start_dim=1, end_dim=-1)\n    \nclass ArcModel(nn.Module):\n    def __init__(self, s=17.0, m=0.5):\n        super().__init__()\n        self.conv = nn.Sequential(\n                            nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, stride=1, padding=1),\n                            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=1, padding=1),\n                            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=5, stride=2, padding=2),\n                            nn.Dropout(0.4),\n                            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1),\n                            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1),\n                            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=2, padding=2),\n                            nn.Dropout(0.4),\n                            Flatten(),\n                            nn.Linear(64*7*7, 128),\n                            nn.ReLU(inplace=True),\n                            nn.BatchNorm1d(128),\n                            nn.Dropout(0.4),\n                        )\n        \n        self.arc = ArcMarginProduct(128,10,s=s,m=m)\n        self.linear = nn.Linear(128, 10)\n        \n    def forward(self, x, y=None):\n        x = self.conv(x)\n#         x = self.arc(x.squeeze(-1).squeeze(-1), y)\n        x =self.linear(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@torch.no_grad()\ndef validation(model, loader, loss_fn):\n    model.eval()\n    losses = []\n    accs = []\n    for x, y in loader: \n        x, y = x.to(device), y.to(device)\n        output = model(x)\n        loss = loss_fn(output, y)\n        losses.append(loss.item())\n        accs.append(accuracy(output.cpu(), y.cpu()))\n    return np.array(losses), np.array(accs)\n\ndef accuracy(output, target, topk=(1,3)):\n    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n    with torch.no_grad():\n        maxk = max(topk)\n        batch_size = target.size(0)\n\n        _, pred = output.topk(maxk, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        res = []\n        for k in topk:\n            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n            res.append(correct_k.mul_(100.0 / batch_size).item())\n        return np.array(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ArcModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\noptimizer = torch.optim.Adam(model.parameters(),\n                  lr=1e-3, betas=(0.9, 0.999), amsgrad=True)\n\n# optimizer = torch.optim.SGD(model.parameters(),\n#                   lr=1e-2, momentum=0.9)\n\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [7, 70], 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.to(device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 100\nfor epoch in range(1, epochs+1):\n    losses = []\n    accs = []\n\n    model.train()\n    for x, y in loader: \n        x, y = x.to(device), y.to(device)\n        optimizer.zero_grad()\n        probs = model(x, y)\n        loss = criterion(probs, y)\n        loss.backward()\n        optimizer.step()\n\n        losses.append(loss.item())\n        accs.append(accuracy(probs.cpu(), y.cpu()))\n\n        del loss, probs, y, x\n        \n    scheduler.step()\n    \n    losses = np.array(losses).mean()\n    accs = np.array(accs).mean(axis=0)\n\n    vlosses, vaccs = validation(model, vloader, criterion)\n    vlosses = vlosses.mean()\n    vaccs = vaccs.mean(axis=0)\n\nprint('Epoch {:3d} -> Train Loss: {:6.3f}, ACC: {:5.2f}%, TOP-3: {:5.2f}%, \\\nValid Loss: {:6.3f}, ACC: {:5.2f}%, TOP-3: {:5.2f}%'\n    .format(epoch, losses, accs[0], accs[1], vlosses, vaccs[0], vaccs[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tds = DataSet(df_test, transform=trfm, mode='test')\ntloader = D.DataLoader(tds, batch_size=batch_size, shuffle=False, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nmodel.eval()\nfor x in tloader: \n    x = x.to(device)\n    probs = model(x)\n    preds.append(probs.detach().cpu().numpy().argmax(axis=1))\n    \npreds = np.concatenate(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = preds\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}