{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Setup"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport shutil\nimport numpy  as np\nimport pandas as pd\n\nimport torch\nimport torch.nn            as nn\nimport torch.nn.functional as F\n\nfrom PIL         import Image\nfrom torchvision import transforms\n\ntorch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.cuda import get_device_properties, get_device_capability, get_device_name, is_available\n\n# GPU availability\nprint(\"Is a GPU available? {}\".format(is_available()))\n\n# Device name\nprint(\"The name of the device is {}\".format(get_device_name()))\n\n# CUDA Version\nprint(\"Device capability is {}\".format(get_device_capability(0)))\n\n# Check available GPU resources\nprint(\"Available GPU memory is {0:.{1}f} GB \".format(get_device_properties(0).total_memory / (1024 ** 3), 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset\nThe full code for the training dataset can be found in the repo.\nFor inference we can make something quickly using a PyTorch dataset to handle batching for us"},{"metadata":{"trusted":true},"cell_type":"code","source":"#%% Loading\n# Load the dataset\ndf = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\n\n# Create a folder\nos.makedirs(\"images\")\n\n#%% Preprocessing\n# To start, there will be no preprocessing as this is very similar to the classic MNIST dataset\n# Possible preprocessing could be to remove empty space around the digits\n# We could also alter pixel values to remove noise\n\n# Bounding Box\ndef bbox(image):\n    \"\"\"\n    Determines the bounding boxes for images to remove empty space where possible\n    :param image:\n    :return:\n    \"\"\"\n    HEIGHT = image.shape[0]\n    WIDTH  = image.shape[1]\n\n    for i in range(image.shape[1]):\n        if (image[:, i] > 0).sum() >= 1:\n            x_min = i - 1 if (i > 1) else 0\n            break\n\n    for i in reversed(range(image.shape[1])):\n        if (image[:, i] > 0).sum() >= 1:\n            x_max = i + 2 if (i < WIDTH - 2) else WIDTH\n            break\n\n    for i in range(image.shape[0]):\n        if (image[i] > 0).sum() >= 1:\n            y_min = i - 1 if (i > 1) else 0\n            break\n\n    for i in reversed(range(image.shape[0])):\n        if (image[i] > 0).sum() >= 1:\n            y_max = i + 2 if (i < HEIGHT - 2) else HEIGHT\n            break\n\n    return x_min, y_min, x_max, y_max\n\n#%% Dataset generation\n# Label and image ID storage\nids    = []\nlabels = []\n\n# Loop through each row\nfor i,row in df.iterrows():\n    # Get the data components\n    id    = \"{}\".format(i)\n    img   = np.reshape(row[1:].values, newshape=(28,28)).astype(np.uint8)\n\n    # Remove empty space\n    x_min, y_min, x_max, y_max = bbox(img)\n    img = img[y_min:y_max, x_min:x_max]\n\n    # Check again\n    x_min, y_min, x_max, y_max = bbox(img)\n    img = img[y_min:y_max, x_min:x_max]\n\n    # Pad\n    img = np.pad(img, (2,2), \"constant\", constant_values=(0,0))\n\n    # Remove any non-maximum pixels\n    img[img >= 50] = 255\n    img[img < 50] = 0\n\n    # Convert and reshape\n    img = Image.fromarray(img)\n    img = img.resize((28,28), Image.ANTIALIAS)\n\n    # Fetch the label\n    label = 0\n\n    # Write to storage\n    img.save(\"images/{}.png\".format(id))\n    ids.append(id)\n    labels.append(label)\n    \n# Save the IDs to a DataFrame and write to CSV\nlabel_df = pd.DataFrame({\"id\": ids, \"label\": labels})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class KannadaMNIST(object):\n    def __init__( self\n                , image_folder    : str\n                , labels          : pd.DataFrame\n                , transforms      : list  = None\n                ):\n        \"\"\"\n        Class for creating a dataset for training the flow chart object detector\n        \"\"\"\n        # Class attributes\n        self.image_folder = image_folder\n        self.labels       = labels\n        self.transforms   = transforms\n\n        # Create an index list\n        self.index = np.arange(self.labels.shape[0])\n\n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self, idx):\n        # Load the labels\n        label = self.labels.iloc[self.index[idx]]\n\n        # Load the image\n        img = Image.open(self.image_folder + \"/\" + label[\"id\"] + \".png\")\n\n        # Apply transformations\n        img_ = self.transforms(img)\n\n        return {\"image\": img_, \"id\":int(label[\"id\"])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch, math, sys\n\n####\n# CODE TAKEN FROM https://github.com/sdoria/SimpleSelfAttention\n####\n\n#Unmodified from https://github.com/fastai/fastai/blob/5c51f9eabf76853a89a9bc5741804d2ed4407e49/fastai/layers.py\ndef conv1d(ni, no, ks=1, stride=1, padding=0, bias=False):\n\t\"Create and initialize a `nn.Conv1d` layer with spectral normalization.\"\n\tconv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias)\n\tnn.init.kaiming_normal_(conv.weight)\n\tif bias: conv.bias.data.zero_()\n\treturn nn.utils.spectral_norm(conv)\n\n# Adapted from SelfAttention layer at https://github.com/fastai/fastai/blob/5c51f9eabf76853a89a9bc5741804d2ed4407e49/fastai/layers.py\n# Inspired by https://arxiv.org/pdf/1805.08318.pdf\nclass SimpleSelfAttention(nn.Module):\n\t\n\tdef __init__(self, n_in, ks=1, sym=False):\n\t\tsuper().__init__()        \n\t\tself.conv = conv1d(n_in, n_in, ks, padding=ks//2, bias=False)            \n\t\tself.gamma = nn.Parameter(torch.Tensor([0.]))      \n\t\tself.sym = sym\n\t\tself.n_in = n_in\n\t\t\n\tdef forward(self, x):\n\t\tif self.sym:\n\t\t\t# symmetry hack by https://github.com/mgrankin\n\t\t\tc = self.conv.weight.view(self.n_in,self.n_in)\n\t\t\tc = (c + c.t())/2\n\t\t\tself.conv.weight = c.view(self.n_in,self.n_in,1)\n\t\t\t\t\n\t\tsize = x.size()  \n\t\tx = x.view(*size[:2],-1)   # (C,N)\n\t\t\n\t\t# changed the order of mutiplication to avoid O(N^2) complexity\n\t\t# (x*xT)*(W*x) instead of (x*(xT*(W*x)))\n\t\t\n\t\tconvx = self.conv(x)   # (C,C) * (C,N) = (C,N)   => O(NC^2)\n\t\txxT = torch.bmm(x, x.permute(0,2,1).contiguous())   # (C,N) * (N,C) = (C,C)   => O(NC^2)  \t\t    \n\t\to = torch.bmm(xxT, convx)   # (C,C) * (C,N) = (C,N)   => O(NC^2)         \n\t\to = self.gamma * o + x  \n\t\t  \n\t\treturn o.view(*size).contiguous()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nNetwork for Kannada MNIST classification\n\"\"\"\n# %% Setup\nimport torch\nimport torch.nn            as nn\nimport torch.nn.functional as F\n\n\n# %% Network\nclass KannadaNet(nn.Module):\n    def __init__(self\n                 , input_shape\n                 , n_class\n                 ):\n        \"\"\"\n        Builds a model for the Kannada MNIST dataset\n        Based on: https://www.kaggle.com/bustam/cnn-in-keras-for-kannada-digits\n        :param input_shape:\n        :param n_class:\n        \"\"\"\n        super(KannadaNet, self).__init__()\n\n        # Build the layers\n        self._build_layers(input_shape=input_shape, n_class=n_class)\n\n        # Initialize weights\n        self._init_weight()\n\n        # Parameter counts\n        print(\"Model contains {} parameters\".format(sum(p.numel() for p in self.parameters() if p.requires_grad)))\n\n    def forward(self, x):\n        x = self._conv_forward(x)\n\n        # Classification\n        x = self.fc_1(x)\n        x = F.relu(x)\n        x = self.bn_1(x)\n\n        x = self.fc_2(x)\n        x = F.relu(x)\n        x = self.bn_2(x)\n\n        confidence = self.confidence(x)\n        out        = self.classifier(x)\n\n        #return out\n        return {\"prediction\":out.cpu().float(), \"confidence\":confidence.cpu().float()}\n\n    def _build_layers(self, input_shape, n_class):\n        \"\"\"\n        Adds layers to the network\n        :param input_shape:\n        :param n_class:\n        :return:\n        \"\"\"\n        # Create the stem of the network\n        self.stem = Conv2D_BN( in_channels =input_shape[0]\n                             , out_channels=32\n                             , padding     =1\n                             )\n\n        # Inception blocks\n        self.inception_res_1 = InceptionBlock(in_shape=[32,28,28], residual=True)\n        self.attention_1 = SimpleSelfAttention(self.inception_res_1.output_shape[0])\n        self.inception_res_2 = InceptionBlock(in_shape=[int(x / y) for x, y in zip(self.inception_res_1.output_shape, [1,2,2])], residual=True)\n        self.attention_2 = SimpleSelfAttention(self.inception_res_2.output_shape[0])\n        self.inception_res_3 = InceptionBlock(in_shape=[int(x / y) for x, y in zip(self.inception_res_2.output_shape, [1, 2, 2])], residual=True)\n        self.attention_3 = SimpleSelfAttention(self.inception_res_3.output_shape[0])\n\n        # Pooling\n        self.pool = nn.MaxPool2d(kernel_size=2)\n\n        # Determine the output dimensions\n        conv_out_dims = self._conv_forward(torch.zeros([1] + input_shape, dtype=torch.float32))\n\n        # Linear layers\n        self.fc_1 = nn.Linear(in_features=conv_out_dims.shape[1], out_features=512)\n        self.bn_1 = nn.BatchNorm1d(num_features=512)\n        self.fc_2 = nn.Linear(in_features=512, out_features=512)\n        self.bn_2 = nn.BatchNorm1d(num_features=512)\n\n        self.confidence = nn.Linear(in_features=512, out_features=1)\n        self.classifier = nn.Linear(in_features=512, out_features=n_class)\n\n\n    # Convolutional forward\n    def _conv_forward(self, x):\n        \"\"\"\n        Forward function for convolutional layers\n        :param x:\n        :return:\n        \"\"\"\n        x = self.stem(x)\n        x = self.inception_res_1(x)\n        x = self.attention_1(x)\n        x = self.pool(x)\n        x = self.inception_res_2(x)\n        x = self.attention_2(x)\n        x = self.pool(x)\n        x = self.inception_res_3(x)\n        x = self.attention_3(x)\n        x = self.pool(x)\n\n        # Flatten\n        x = torch.flatten(x, start_dim=1)\n\n        return x\n\n    # Weight initialization\n    def _init_weight(self):\n        for m in self.modules():\n            print(m)\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1.0)\n                m.bias.data.zero_()\n\n# %% Utility functions\nclass InceptionBlock(nn.Module):\n    def __init__(self, in_shape, residual=True):\n        super(InceptionBlock, self).__init__()\n        # Class parameters\n        self.residual = residual\n\n        # Define the component operations\n        self.conv_1x1_1 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))\n        self.bn_1       = nn.BatchNorm2d(num_features=self.conv_1x1_1.out_channels)\n\n        self.conv_1x1_2 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]))\n        self.bn_2       = nn.BatchNorm2d(num_features=self.conv_1x1_2.out_channels)\n\n        self.conv_1x1_3 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))\n        self.bn_3       = nn.BatchNorm2d(num_features=self.conv_1x1_3.out_channels)\n\n        self.conv_1x1_4 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))\n        self.bn_4       = nn.BatchNorm2d(num_features=self.conv_1x1_4.out_channels)\n\n\n        self.conv_3x3_1 = nn.Conv2d(kernel_size=3, in_channels=self.conv_1x1_2.out_channels, out_channels=self.conv_1x1_1.out_channels * 2, padding=1)\n        self.bn_5       = nn.BatchNorm2d(num_features=self.conv_3x3_1.out_channels)\n\n        self.conv_3x3_2 = nn.Conv2d(kernel_size=3, in_channels=self.conv_1x1_3.out_channels, out_channels=int(in_shape[0]/4), padding=1)\n        self.bn_6       = nn.BatchNorm2d(num_features=self.conv_3x3_2.out_channels)\n\n        self.conv_3x3_3 = nn.Conv2d(kernel_size=3, in_channels=self.conv_3x3_2.out_channels, out_channels=int(in_shape[0]/4), padding=1)\n        self.bn_7       = nn.BatchNorm2d(num_features=self.conv_3x3_3.out_channels)\n\n        self.pool_3x3   = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)\n\n        # Determine the block output shape\n        self.output_shape = self._get_out_shape(in_shape)\n\n    def forward(self, x):\n        # First path\n        x_1 = self.conv_1x1_1(x)\n        x_1 = F.relu(x_1)\n        x_1 = self.bn_1(x_1)\n\n\n        # Second path\n        x_2 = self.conv_1x1_2(x)\n        x_2 = F.relu(x_2)\n        x_2 = self.bn_2(x_2)\n\n        x_2 = self.conv_3x3_1(x_2)\n        x_2 = F.relu(x_2)\n        x_2 = self.bn_5(x_2)\n\n        \n        # Third path\n        x_3 = self.conv_1x1_3(x)\n        x_3 = F.relu(x_3)\n        x_3 = self.bn_3(x_3)\n\n        x_3 = self.conv_3x3_2(x_3)\n        x_3 = F.relu(x_3)\n        x_3 = self.bn_6(x_3)\n\n        x_3 = self.conv_3x3_3(x_3)\n        x_3 = F.relu(x_3)\n        x_3 = self.bn_7(x_3)\n\n        \n        # Fourth path\n        x_4 = self.pool_3x3(x)\n        x_4 = self.conv_1x1_4(x_4)\n        x_4 = F.relu(x_4)\n        x_4 = self.bn_4(x_4)\n\n\n        # Concatenate\n        if self.residual:\n            x = torch.cat([x_1, x_2, x_3, x_4, x], dim=1)\n        else:\n            x = torch.cat([x_1, x_2, x_3, x_4], dim=1)\n\n        return x\n\n    def _get_out_shape(self, input_shape):\n        return list(self.forward(torch.zeros([1] + input_shape)).shape[1:])\n\n\nclass Conv2D_BN(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel=3, stride=1, padding=1, activation=True):\n        \"\"\"\n        Convolution followed by Batch Normalization, with optional Mish activation\n        :param in_features:\n        :param out_features:\n        :param kernel_size:\n        :param stride:\n        :param padding:\n        :param groups:\n        :param activation:\n        \"\"\"\n        super(Conv2D_BN, self).__init__()\n\n        # Class parameters\n        self.activation = activation\n\n        # Convolution and BatchNorm class\n        self.conv = nn.Conv2d( in_channels =in_channels\n                             , out_channels=out_channels\n                             , kernel_size =kernel\n                             , stride      =stride\n                             , padding     =padding\n                             , bias        =True\n                             )\n\n        self.bn = nn.BatchNorm2d(num_features=out_channels, momentum=0.9)\n\n    def forward(self, x):\n        x = self.conv(x)\n        if self.activation:\n            x = F.relu(x)\n        x = self.bn(x)\n\n\n        return x\n\n\ndef block_forward(layers, x):\n    for layer in layers:\n        x = layer(x)\n\n    return x\n\n\n# Mish Activation\ndef Mish(x):\n    r\"\"\"\n    Mish activation function is proposed in \"Mish: A Self\n    Regularized Non-Monotonic Neural Activation Function\"\n    paper, https://arxiv.org/abs/1908.08681.\n    \"\"\"\n\n    return x * torch.tanh(F.softplus(x))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Assembly\nConvert the dataset into arrays, create a dataloader, instantiate the network, and load the weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the dataset\ndataset = KannadaMNIST(image_folder=\"images\", labels=label_df, transforms=transforms.ToTensor())\n\n# Create a dataloader\nloader = torch.utils.data.DataLoader( dataset    =dataset\n                                    , batch_size =256\n                                    , shuffle    =False\n                                    #, num_workers=4\n                                    , pin_memory =True\n                                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the loader\nfor sample in loader:\n    test = sample[\"image\"][0]\n    break\n    \ntest = transforms.functional.to_pil_image(test)\ntest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Instantiate and load a model\nnet = KannadaNet( input_shape=[1, 28, 28]\n                , n_class    =10\n                )\nnet.load_state_dict(torch.load(\"../input/pytorch-kannada-v2/CustomNetV2 Submission Weights.tar\"), strict=True)\nnet.cuda()\nnet.eval()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Inference\npredictions = []\nids         = []\nwith torch.no_grad():\n    for x in loader:\n        predictions.append(F.softmax(net(x[\"image\"].cuda())[\"prediction\"], dim=1).cpu().argmax(dim=1))\n        ids.append(x[\"id\"])\n\npredictions = torch.cat(predictions)\nids         = torch.cat(ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the sample and update the predictions\nlabel_df['label'] = predictions\nlabel_df['id']    = ids\n\nlabel_df = label_df.sort_values(\"id\")\n\n# Check the submission\nprint(label_df.head())\n\n# Write the submission\nlabel_df.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clean up\nshutil.rmtree(\"images\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}