{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nfrom scipy.signal import correlate2d, convolve2d\n\nimport numpy as np\n\nfrom IPython.display import clear_output\nimport PIL\nimport torch\nimport torchvision\nfrom torchvision import datasets, transforms\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\n%matplotlib inline\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.optim as optim\n\nfrom IPython.display import clear_output\n\nfrom collections import defaultdict\n\nimport seaborn as sn\nimport pandas as pd\n#from torchsummary import summary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_classify(img, ps):\n    ps = ps.cpu().data.numpy().squeeze()\n\n    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze(), cmap='gray')\n    ax1.axis('off')\n    ax2.barh(np.arange(10), ps)\n    ax2.set_aspect(0.1)\n    ax2.set_yticks(np.arange(10))\n    ax2.set_yticklabels(np.arange(10))\n    ax2.set_title('class probability')\n    ax2.set_xlabel('probability')\n    ax2.set_ylabel('class')\n    ax2.set_xlim(0, 1.1)\n\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_features(features):\n    if len(features.shape) < 4:\n        batch, num_feature = features.shape[:2]\n        for i, feature in enumerate(features):\n            plt.subplot(1, num_feature, i+1)\n            plt.imshow(feature.numpy().transpose(1,2,0))\n    else:\n        batch, num_feature = features.shape[:2]\n        for i, element in enumerate(features):\n            for j, feature in enumerate(element):\n                plt.subplot(batch, num_feature, i * num_feature + j + 1)\n                plt.imshow(feature.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest=pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tt_data=pd.concat([train, test], ignore_index=True)\n# tt_data=tt_data.sample(frac=1)\n# test=tt_data.sample(frac=0.1)\n# train=tt_data.drop(test.index)\n# test=test.reset_index(drop=True)\n# train=train.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train',train.shape)\nprint('test',test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=train.drop('label',axis=1)\ntrain_targets=train['label']\n\ntest_data=test.drop('label',axis=1)\ntest_targets=test['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=torch.from_numpy(train_data.values).float().view(train_data.shape[0],28,28)\ntrain_targets=torch.from_numpy(train_targets.values).long().view(train_data.shape[0])\n\ntest_data=torch.from_numpy(test_data.values).float().view(test_data.shape[0],28,28)\ntest_targets=torch.from_numpy(test_targets.values).long().view(test_data.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num, height, width = np.array(train_data).shape\nimg_min = np.array(train_data).min()\nimg_max = np.array(train_data).max()\nimg_norm_mean = np.array(train_data, dtype=float).mean() / img_max\nimg_std = np.sqrt(np.sum((np.array(train_data) / img_max  - img_norm_mean) ** 2) / (num * height * width))\nprint(img_min, img_max, img_norm_mean, img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train set shape: ', train_data.shape)\nprint('train labels shape: ', train_targets.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('test set shape: ', test_data.shape)\nprint('test labels shape: ', test_targets.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('train dataset distribution')\nplt.hist(train_targets);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('test dataset distribution')\nplt.hist(test_targets);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class KannadaDataSet(torch.utils.data.Dataset):\n    def __init__(self, images, labels, transforms = None):\n        self.images = images\n        self.labels = labels\n        self.transforms = transforms\n         \n    def __len__(self):\n        return self.images.shape[0]\n    \n    def __getitem__(self, i):\n        data = np.array(self.images.iloc[i,:]).astype(np.uint8).reshape(28,28,1)\n        \n        if self.transforms:\n            data = self.transforms(data)\n            \n        return (data, self.labels[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_orig_T = transforms.Compose(([\n    transforms.ToTensor(),\n#     transforms.Normalize((img_norm_mean,), (img_std,))\n]))\ntrain_aug_T = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.RandomCrop(28),\n    transforms.RandomAffine(degrees=10, translate=(0.1, 0.1)),\n    transforms.ToTensor(),\n#     transforms.Normalize((img_norm_mean,), (img_std,))\n]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_T = transforms.Compose(([\n    transforms.ToTensor(),\n#     transforms.Normalize((img_norm_mean,), (img_std,))\n]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orig_train_set=KannadaDataSet(train.drop('label',axis=1), train['label'],train_orig_T)\naug_train_set=KannadaDataSet(train.drop('label',axis=1), train['label'],train_aug_T)\ntrain_set = torch.utils.data.ConcatDataset([orig_train_set,aug_train_set])\ntest_set=KannadaDataSet(test.drop('label',axis=1), test['label'], test_T) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(train_set, \n                                           batch_size=128, \n                                           shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(test_set,\n                                          batch_size=128, \n                                          shuffle=False)\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, labels = next(iter(train_loader))\nimages, labels = images[:8], labels[:8]\nplt.imshow(torchvision.utils.make_grid(images)[0,:,:], cmap='gray')\nprint(labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LeNet - 5"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=1, \n                               out_channels=12, \n                               kernel_size=5, \n                               padding=0)\n        self.bn1 = nn.BatchNorm2d(12)\n        self.conv2 = nn.Conv2d(in_channels=12, \n                               out_channels=32, \n                               kernel_size=3, \n                               padding=0)\n        self.bn2 = nn.BatchNorm2d(32)\n        self.fc1 = nn.Linear(32 * 5 * 5, 240)\n        self.fc2 = nn.Linear(240, 168)\n        self.fc3 = nn.Linear(168, 10)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = F.relu(x)\n        #x = F.max_pool2d(x,2)\n        x = F.avg_pool2d(x,2)\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = F.relu(x)\n        #x = F.max_pool2d(x,2)\n        x = F.avg_pool2d(x,2)\n        x = x.view(-1, 32 * 5 * 5)\n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        x = F.relu(x)\n        x = self.fc3(x)\n        x = F.log_softmax(x, dim=1)\n        return x\n    \n    def features_2(self, x):\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n#         x = F.max_pool2d(F.relu(self.conv1(x)),2)\n#         x = F.max_pool2d(F.relu(self.conv2(x)),2)\n        return x\n    \n    def features_1(self, x):\n        x = F.relu(self.conv1(x))\n#         x = F.max_pool2d(F.relu(self.conv1(x)),2)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = Net().to(device)\n# summary(net, (1, 28, 28))\nnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nshow_features(net.features_1(images.to(device)).detach().cpu())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nshow_features(net.features_2(images.to(device)).detach().cpu())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9)\n\nepoch=0\n\ncurve_x=[]\nloss_curve_y=[]\nv_loss_curve_y=[]\nacc_curve_y=[]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"num_epochs=5\n\ne=0\nwhile e < num_epochs:\n    epoch+=1\n    e+=1\n    \n#     if e == 3:\n#         optimizer.param_groups[0]['lr'] = 1e-5\n\n#     if e == 5:\n#         optimizer.param_groups[0]['lr'] = 1e-8\n        \n    running_loss = 0.0\n    for i, data in enumerate(train_loader, 0):\n\n        inputs, labels = data[0].to(device), data[1].to(device)\n\n        optimizer.zero_grad()\n\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n        \n        if i != 0 and i % 100 == 0:    \n            validation_loss = 0.\n            correct = 0\n            total = 0\n\n            with torch.no_grad():\n                net.eval()\n\n                for v_data in test_loader:\n                    v_inputs, v_labels = v_data[0].to(device), v_data[1].to(device)\n\n                    v_outputs = net(v_inputs)\n                    v_loss = criterion(v_outputs, v_labels)\n\n                    validation_loss+=v_loss.item()\n\n                    _, predicted = torch.max(v_outputs.data, 1)\n                    total += v_labels.size(0)\n                    correct += (predicted == v_labels).sum().item()\n            net.train()\n        #     clear_output()              \n            print('epoch %d, step %5d training loss: %.3f validation loss: %.3f test acc: %.3f' %\n              (epoch, i, running_loss / 1000, validation_loss/1000, 100 * correct / total))\n            \n            curve_x.append(len(curve_x))\n            loss_curve_y.append(running_loss/1000)\n            v_loss_curve_y.append(validation_loss/1000)\n            acc_curve_y.append(100 * correct / total)\n            \n            running_loss = 0.0\n\nprint('finish')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# training curves"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(curve_x,v_loss_curve_y, label='validation')\naxes[0].plot(curve_x,loss_curve_y, '--',label='train')\naxes[0].set_title('validation loss')\naxes[0].set_xlabel(\"iterations\")\naxes[0].set_ylabel(\"loss\")\naxes[0].set_xticks([])\naxes[0].legend()\naxes[1].plot(curve_x,acc_curve_y)\naxes[1].set_title('test acc')\naxes[1].set_xlabel(\"iterations\")\naxes[1].set_ylabel(\"acc\")\naxes[1].set_xticks([])\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nshow_features(net.features_1(images.to(device)).detach().cpu())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nshow_features(net.features_2(images.to(device)).detach().cpu())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, labels = next(iter(test_loader))\nimages, labels = images.to(device), labels.to(device)\nimages, labels = images[:8], labels[:8]\noutputs = net(images)\n_, predicted = torch.max(outputs, 1)\n\nplt.imshow(torchvision.utils.make_grid(images.cpu())[0,:,:], cmap='gray')\nprint('gt:', labels)\nprint('predict:', predicted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correct = 0\ntotal = 0\n\nerrors_imgs=[]\nerrors_labels=[]\nconfusion_matrix = torch.zeros(10, 10)\nerr_confusion_matrix = torch.zeros(10, 10)\ncorrect_samples={i:None for i in range(10)}\nwith torch.no_grad():\n    for data in test_loader:\n        images, labels = data[0].to(device), data[1].to(device)\n        outputs = net(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        errors_imgs.extend(images[predicted != labels])\n        errors_labels.extend(zip(predicted[predicted != labels], labels[predicted != labels]))\n        \n        for i in [i for i in correct_samples if type(correct_samples[i]) != torch.Tensor]:\n            ci = images[(predicted == labels) & (i == labels)]\n            if ci.shape[0] > 0:\n                correct_samples[i] = ci[0]\n        for t, p in zip(labels.view(-1), predicted.view(-1)):\n            confusion_matrix[t.long(), p.long()] += 1\n            if t.long() != p.long():\n                err_confusion_matrix[t.long(), p.long()] += 1\nerrors_imgs=torch.stack(errors_imgs)\nerrors_labels=np.array([(p.item(), t.item())for p, t in errors_labels])\nprint('accuracy of the network on the test images: %d %%' % (100 * correct / total))\n#torch.save(net, 'torch_kannada_mnist_model.pt')  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('errors: ',len(errors_labels))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# heatmap"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_cm = pd.DataFrame(confusion_matrix.numpy(), range(10), range(10))\n\nplt.figure(figsize = (20,20))\nsn.set(font_scale=2)\nsn.heatmap(df_cm, annot=True,annot_kws={\"size\": 14}, fmt='g')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# errors heatmap"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_cm = pd.DataFrame(err_confusion_matrix.numpy(), range(10), range(10))\n\nplt.figure(figsize = (20,20))\nsn.set(font_scale=2)\nsn.heatmap(df_cm, annot=True,annot_kws={\"size\": 14}, fmt='g')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sn.set(font_scale=1)\nsn.set_style(\"whitegrid\", {'axes.grid' : False})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f=lambda i, a: ({k: len(v) for k, v in a.items()} if [a[x].append(i) for x in i] else {})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"errors=f(errors_labels[:,1],defaultdict(list))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# errors distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6,4))\nplt.barh(*zip(*errors.items()))\nplt.yticks(range(10))\nplt.xlabel('erorrs')\nplt.ylabel('labels')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# errors features"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nshow_features(net.features_1(errors_imgs[:8]).detach().cpu())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nshow_features(net.features_2(errors_imgs[:8]).detach().cpu())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# errors examples"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"for img_indx in range(10):\n    pred_digit=errors_labels[img_indx][0].item()\n    true_digit=errors_labels[img_indx][1].item()\n    print('PREDICTED: ', pred_digit, '!=','TRUE: ', true_digit)\n    ps = net.forward(errors_imgs[img_indx].unsqueeze(0))\n    view_classify(errors_imgs[img_indx].cpu(), torch.softmax(ps,dim=1))\n    plt.show()\n    \n    print(f'FEATURES_1 OF FALSE {true_digit}')\n    plt.figure(figsize=(15,10))\n    show_features(net.features_1(errors_imgs[img_indx:img_indx+1].to(device)).detach().cpu())\n    plt.show()\n\n    print(f'FEATURES_1 OF TRUE {true_digit}')\n    plt.figure(figsize=(15,10))\n    show_features(net.features_1(correct_samples[true_digit].unsqueeze(0).to(device)).detach().cpu())\n    plt.show()\n    \n    print(f'FEATURES_1 OF TRUE {pred_digit}')\n    plt.figure(figsize=(15,10))\n    show_features(net.features_1(correct_samples[pred_digit].unsqueeze(0).to(device)).detach().cpu())\n    plt.show()\n    \n    print(f'FEATURES_2 OF FALSE {true_digit}')\n    plt.figure(figsize=(15,10))\n    show_features(net.features_2(errors_imgs[img_indx:img_indx+1].to(device)).detach().cpu())\n    plt.show()\n    \n    print(f'FEATURES_2 OF TRUE {true_digit}')\n    plt.figure(figsize=(15,10))\n    show_features(net.features_2(correct_samples[true_digit].unsqueeze(0).to(device)).detach().cpu())\n    plt.show()\n    \n    print(f'FEATURES_2 OF TRUE {pred_digit}')\n    plt.figure(figsize=(15,10))\n    show_features(net.features_2(correct_samples[pred_digit].unsqueeze(0).to(device)).detach().cpu())\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# kaggle submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# net=torch.load('torch_kannada_mnist_model.pt')\n# net.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle=pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle=kaggle.drop('id', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle=torch.from_numpy(kaggle.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle=kaggle.view(kaggle.shape[0],28,28).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# T = transforms.Compose([\n#     transforms.Normalize((img_norm_mean,), (img_std,))\n# ])\n# kaggle=T(kaggle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions=[]\nwith torch.no_grad():\n    for i, data in enumerate(kaggle):\n        images = data.to(device)\n        outputs = net(images.float().unsqueeze(0).unsqueeze(0))\n        _, predicted = torch.max(outputs.data, 1)\n        predictions.append([i,predicted.item()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions=pd.DataFrame(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions.columns=['id','label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}