{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np \nimport pandas as pd \nimport os\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom torchvision.transforms import ToTensor\nfrom torchvision.models import resnet as model_res\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm import tqdm\n        \nprint(os.listdir('/kaggle/input/Kannada-MNIST'))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dir_csv = '/kaggle/input/Kannada-MNIST'\ndir_train_img = dir_csv +'/train.csv'\ndir_test_img = dir_csv + '/test.csv'\n\ntrain = pd.read_csv(dir_train_img)\ntest = pd.read_csv(dir_test_img)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"n_classes = 10\nn_epochs = 2\nBATCH_SIZE = 256\nIMG_DIM = 28\n\n# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(dir_train_img)\ndf_test = pd.read_csv(dir_test_img)\n\ntarget = df_train['label']\ndf_train.drop('label', axis=1, inplace=True)\n\nX_test = pd.read_csv('../input/Kannada-MNIST/test.csv')\nX_test.drop('id', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_dev, y_train, y_dev = train_test_split(df_train, target, stratify=target, random_state=42, test_size=0.01)\nprint('X_train', len(X_train))\nprint('X_dev', len(X_dev))\nprint('X_test', len(X_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=10, ncols=10, figsize=(10,10))\n\n# I know these for loops look weird, but this way num_i is only computed once for each class\nfor i in range(10): # Column by column\n    num_i = X_train[y_train == i]\n    ax[0][i].set_title(i)\n    for j in range(10): # Row by row\n        ax[j][i].axis('off')\n        ax[j][i].imshow(num_i.iloc[j, :].to_numpy().astype(np.uint8).reshape(28, 28), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Infer from standard PyTorch torch.utils.data.Dataset class\nclass CharData(Dataset):\n    def __init__(self, images, labels, transform, classes):\n        self.X = images\n        #for i in range(0, len(self)):\n        #    self.X[i] = np.array(self.X[i]).reshape((IMG_DIM,IMG_DIM,1))\n        self.y = labels\n        self.tranform = transform\n        self.classes = classes\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx=None):\n        img = np.array(self.X.iloc[idx,:], dtype='uint8').reshape((IMG_DIM,IMG_DIM,1))\n        if self.y is not None:\n            y = np.zeros(self.classes, dtype='float32')\n            y[self.y.iloc[idx]] = 1\n            return img, y\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ToTensor_customized(ToTensor):\n    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n\n    def __call__(self, sample):\n        image = sample['X']\n        #image = image.permute((0,1, 2))\n        if sample['y'] is not None:\n            label = sample['y']\n        else:\n            return {'X': torch.from_numpy(image)}\n        # swap color axis because\n        # numpy image: H x W x C\n        # torch image: C X H X W\n        \n        return {'X': torch.from_numpy(image),\n                'y': torch.from_numpy(label)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Put some augmentation on training data using pytorch torchvision.transforms\n\ntrain_transform = transforms.Compose([\n    transforms.Resize([IMG_DIM,IMG_DIM]),\n    #transforms.ToPILImage(),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=5),\n    transforms.ToTensor()\n])\n\n# test data is not augmented, kept as it is.\ntest_transform = transforms.Compose([\n    transforms.Resize([IMG_DIM,IMG_DIM]),\n    #transforms.ToPILImage(),\n    transforms.ToTensor()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Dataset objects\n\ntrain_dataset = CharData(images=X_train, labels=y_train, transform=train_transform, classes=10)\ndev_dataset = CharData(images=X_dev, labels=y_dev, transform=test_transform, classes=10)\ntest_dataset = CharData(images=X_test, labels=None, transform=test_transform, classes=10)\n\n#print(len(train_dataset))\n#print(np.shape(train_dataset[0][0]))\n\n#train_dataset.unsqueeze_(1)\n#train_dataset.y.unsqueeze_(0)\n#dev_dataset.unsqueeze_(1)\n#test_dataset.unsqueeze_(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defining the data generators for producing batches of data\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\nprint(np.shape(train_dataset.X))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the network by inheriting nn.Module\n\nDEPTH_MULT = 2\n\nclass ConvLayer(nn.Module):\n    def __init__(self, in_channel, out_channel, kernel_size=3):\n        super(ConvLayer, self).__init__()\n        self.ops = nn.Sequential(\n            nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, stride=1, padding=kernel_size//2),\n            nn.BatchNorm2d(out_channel),\n            nn.ReLU(inplace=True)\n        )    \n    def forward(self, x):\n        return self.ops(x)\n    \nclass FCLayer(nn.Module):\n    def __init__(self, in_features, out_features):\n        super(FCLayer, self).__init__()\n        self.ops = nn.Sequential(\n            nn.Linear(in_features, out_features),\n            nn.BatchNorm1d(out_features),\n            nn.ReLU(inplace=True)\n        )\n    def forward(self, x):\n        #print(x.shape)\n        return self.ops(x)\n\nclass Flatten(nn.Module):\n    def __init__(self):\n        super(Flatten, self).__init__()\n    def forward(self, x):\n        #print(x.shape)\n        shape = torch.prod(torch.tensor(x.shape[1:])).item()\n        return x.view(-1, shape)\n        #return x.view(shape, -1)        \n    \nclass Net(nn.Module):\n    def __init__(self, num_classes):\n        super(Net, self).__init__()\n        self.conv1 = ConvLayer(1, DEPTH_MULT * 32)\n        self.conv2 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        self.conv3 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        self.conv3 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        self.conv4 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        \n        self.conv5 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 64)\n        self.conv6 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv7 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv8 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv9 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv10 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        \n        self.mp = nn.MaxPool2d(kernel_size=2, stride=2)\n        \n        self.flat = Flatten()\n        \n        #self.fc1 = FCLayer(DEPTH_MULT * 64 * 7 * 7, DEPTH_MULT * 512)\n        self.fc1 = FCLayer(25088, DEPTH_MULT * 512)\n        self.fc2 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)\n        self.fc3 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)\n        self.fc4 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)\n        self.projection = nn.Linear(DEPTH_MULT * 512, num_classes)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n        x = self.conv5(x)\n        x = self.conv6(x)\n        x = self.conv7(x)\n        x = self.conv8(x)\n        x = self.conv9(x)\n        x = self.conv10(x)\n        \n        x = self.mp(x)\n        \n        x = self.flat(x)\n        \n        x = self.fc1(x)\n        x = self.fc2(x)\n        x = self.fc3(x)\n        x = self.fc4(x)\n        x = self.projection(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Net(10)\nmodel = model.to(device)\n\nn_epochs = 6\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=n_epochs // 4, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def criterion(input, target, size_average=True):\n    \"\"\"Categorical cross-entropy with logits input and one-hot target\"\"\"\n    l = -(target * torch.log(F.softmax(input, dim=1) + 1e-10)).sum(1)\n    if size_average:\n        l = l.mean()\n    else:\n        l = l.sum()\n    return l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(epochs, history=None):\n    model.train()\n    \n    for batch_idx, (data, target) in enumerate(train_loader):\n        data = data.to(device)\n        target = target.to(device)\n        \n        optimizer.zero_grad()\n        #print(np.shape(data))\n        data = data.permute(0, 3, 1, 2).float()\n        output = model(data)\n        loss = criterion(output, target)\n        \n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n        \n        loss.backward()\n        optimizer.step()\n        \n        if (batch_idx + 1) % 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLR: {:.6f}\\tLoss: {:.6f}'.format(\n                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n                100. * (batch_idx + 1) / len(train_loader),\n                optimizer.state_dict()['param_groups'][0]['lr'],\n                loss.data))\n                \n    exp_lr_scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(epoch, history=None):\n    model.eval()\n    loss = 0\n    correct = 0\n    \n    with torch.no_grad():\n        for data, target in dev_loader:\n            data = data.to(device)\n            target = target.to(device)\n\n            data = data.permute(0, 3, 1, 2).float()\n            output = model(data)\n\n            loss += criterion(output, target, size_average=False).data\n\n            pred = output.data.max(1, keepdim=True)[1]\n            correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()\n    \n    loss /= len(dev_loader.dataset)\n    accuracy = correct / len(dev_loader.dataset)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()\n        history.loc[epoch, 'dev_accuracy'] = accuracy\n    \n    print('Dev loss: {:.4f}, Dev accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(dev_loader.dataset),\n        100. * accuracy))     \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\nhistory = pd.DataFrame()\n\nfor epoch in range(n_epochs):\n    print('Epoch: '+ str(epoch))\n    torch.cuda.empty_cache()\n    gc.collect()\n    train(epoch, history)\n    evaluate(epoch, history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history['train_loss'].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.dropna()['dev_loss'].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.dropna()['dev_accuracy'].plot();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\npredictions = []\n\nfor data in tqdm(test_loader):\n    data = data.to(device)\n    data = data.permute(0, 3, 1, 2).float()\n    output = model(data).max(dim=1)[1] # argmax\n    predictions += list(output.data.cpu().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}