{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Preparation"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nfrom pathlib import Path\nfrom PIL import Image\nimport random\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport torch\nimport torchvision\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = {\n    # Batch Size for Training and Varidation\n        \"batch_size\": 1024,\n    # CUDA:0 or CPU\n        \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n    # Epoch Size for Training and Validation\n        \"epoch_size\": 10,\n    # Ratio of Filling Noise on Training and Validation Images\n        \"noise_ratio\": 0.25,\n    # Sigma Parameter of Gauss Deviation for Transform\n        \"noise_sigma\": 0.1,\n    # Path to Dig-MNIST.csv\n        \"path_Dig-MNIST_csv\": Path(\"../input/Kannada-MNIST/Dig-MNIST.csv\"),\n    # Path to test.csv\n        \"path_test_csv\": Path(\"../input/Kannada-MNIST/test.csv\"),\n    # Path to train.csv\n        \"path_train_csv\": Path(\"../input/Kannada-MNIST/train.csv\"),\n    # Range of Degrees Rotated by RandomRotation\n        \"pil_trans_degree\": (-10, 10),\n    # Range of Aspect Ratio of the Origin Aspect Ratio Cropped by RandomResizedCrop\n        \"pil_trans_ratio\": (0.8*0.8, 1.25*1.25),\n    # Range of Size of the Origin Size Cropped by RandomResizedCrop\n        \"pil_trans_scale\": (0.75*0.75, 1.0),\n    # Random Seed\n        \"seed\": 17122019,\n    # Ratio of Training Dataset against Overall One\n        \"train_dataset_ratio\": 0.9,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"random.seed(cfg[\"seed\"])\nnp.random.seed(cfg[\"seed\"])\ntorch.manual_seed(cfg[\"seed\"])\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(cfg[\"seed\"])\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sns.set(style=\"darkgrid\", context=\"notebook\", palette=\"muted\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class KannadaMNISTDataset(torch.utils.data.Dataset):\n    def __init__(self,\n                 path_csv: Path,\n                 cfg: dict,\n                 transform=None):\n        df_csv = pd.read_csv(path_csv)\n        self.imgs = df_csv.drop([\"label\"], axis=1).values.astype(np.int32)\n        # Reshape Image from (data_size, 784) to (data_size, 1, 28, 28)\n        self.imgs = self.imgs.reshape(-1, 1, 28, 28)\n        self.labels = torch.tensor(df_csv[\"label\"],\n                                   dtype=torch.int64,\n                                   device=cfg[\"device\"])\n        self.transform = transform\n        self.device = cfg[\"device\"]\n\n    def __len__(self):\n        return self.labels.shape[0]\n\n    def __getitem__(self, idx):\n        img = self.imgs[idx]\n        label = self.labels[idx]\n        if self.transform is None:\n            # Scale Image from [0, 255] to [0.0, 1.0]\n            img = torch.tensor(img/255.0,\n                               dtype=torch.float32,\n                               device=self.device)\n        else:\n            img = self.transform(img)\n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class KannadaMNISTTransform():\n    def __init__(self, cfg: dict):\n        self.device = cfg[\"device\"]\n        self.noise_ratio = cfg[\"noise_ratio\"]\n        self.noise_sigma = cfg[\"noise_sigma\"]\n        self.pil_trans = torchvision.transforms.Compose([\n            torchvision.transforms.ToPILImage(),\n            torchvision.transforms.RandomResizedCrop(28,\n                                                     scale=cfg[\"pil_trans_scale\"],\n                                                     ratio=cfg[\"pil_trans_ratio\"]),\n            torchvision.transforms.RandomRotation(degrees=cfg[\"pil_trans_degree\"]),\n            torchvision.transforms.ToTensor()\n        ])\n\n    def __call__(self, img: np.ndarray):\n        # Add Noise on Images\n        mask = np.random.random(img.shape)>self.noise_ratio\n        noise = np.random.normal(0.0,\n                                 self.noise_sigma,\n                                 size=img.shape)\n        noise[mask] = 0.0\n        noise *= 255.0\n        noise = noise.astype(np.int32)\n        img += noise\n\n        # Execute Pillow's Transforms\n        img = self.pil_trans(img[0])\n\n        # Scale Image from [0, 255] to [0.0, 1.0]\n        img = img.to(torch.float32)/255.0\n\n        return img.to(self.device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def create_training_datasets(cfg: dict,\n                             transform: KannadaMNISTTransform):\n    # Create Overall Dataset Setting KannadaMNISTTransform\n    overall_dataset = KannadaMNISTDataset(cfg[\"path_train_csv\"], cfg, transform)\n    # Split Overall Dataset into Training and Validation Ones\n    train_size = int(len(overall_dataset) * cfg[\"train_dataset_ratio\"])\n    valid_size = len(overall_dataset) - train_size\n    train_dataset, valid_dataset = torch.utils.data.random_split(overall_dataset,\n                                                                 [train_size, valid_size])\n    return train_dataset, valid_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Training Datasets\ntrain_dataset, valid_dataset = create_training_datasets(cfg,\n                                                        KannadaMNISTTransform(cfg))\n# Learning Dataset\nlrn_dataset = KannadaMNISTDataset(cfg[\"path_train_csv\"], cfg, None)\n# Investigation Dataset\ninv_dataset = KannadaMNISTDataset(cfg[\"path_Dig-MNIST_csv\"], cfg, None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Learning"},{"metadata":{},"cell_type":"markdown","source":"We define the VGG-based Network including batch norm layers."},{"metadata":{"trusted":true},"cell_type":"code","source":"class ThisNetwork(torch.nn.Module):\n    def __init__(self):\n        super(ThisNetwork, self).__init__()\n        self.features = torch.nn.Sequential(\n            # (batch,1,28,28) -> (batch,64,28,28)\n            torch.nn.Conv2d(in_channels=1,\n                            out_channels=64,\n                            kernel_size=3,\n                            padding=1),\n            torch.nn.BatchNorm2d(num_features=64),\n            torch.nn.ReLU(inplace=True),\n            # (batch,64,28,28) -> (batch,64,14,14)\n            torch.nn.MaxPool2d(kernel_size=2,\n                               stride=2),\n            # (batch,64,14,14) -> (batch,128,14,14)\n            torch.nn.Conv2d(in_channels=64,\n                            out_channels=128,\n                            kernel_size=3,\n                            padding=1),\n            torch.nn.BatchNorm2d(num_features=128),\n            torch.nn.ReLU(inplace=True),\n            # (batch,128,14,14) -> (batch,128,7,7)\n            torch.nn.MaxPool2d(kernel_size=2,\n                               stride=2),\n        )\n        self.avgpool = torch.nn.AdaptiveAvgPool2d(output_size=3)\n        self.classifier = torch.nn.Sequential(\n            # (batch,1152) -> (batch,256)\n            torch.nn.Linear(in_features=1152,\n                            out_features=256),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Dropout(p=0.5),\n            # (batch,256) -> (batch,256)\n            torch.nn.Linear(in_features=256,\n                            out_features=256),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Dropout(p=0.5),\n            # (batch,256) -> (batch,10)\n            torch.nn.Linear(in_features=256,\n                            out_features=10),\n        )\n        self.log_softmax = torch.nn.LogSoftmax(dim=-1)\n\n    def forward(self, x):\n        # (batch,1,28,28) -> (batch,128,7,7)\n        x = self.features(x)\n        # (batch,128,7,7) -> (batch,128,3,3)\n        x = self.avgpool(x)\n        # (batch,128,3,3) -> (batch,1152)\n        x = x.view(x.size(0), -1)\n        # (batch,1152) -> (batch,10)\n        x = self.classifier(x)\n        # (batch,10) -> (batch,10)\n        x = self.log_softmax(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"network = ThisNetwork().to(cfg[\"device\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def learn(network: torch.nn.Module,\n          train_dataset: KannadaMNISTDataset,\n          valid_dataset: KannadaMNISTDataset,\n          cfg: dict):\n    result = {\"Epoch\" : [],\n              \"Type\" : [],\n              \"Average Loss\" : [],\n              \"Accuracy\" : []}\n    criterion = torch.nn.NLLLoss()\n    optimizer = torch.optim.Adam(network.parameters())\n    train_loader = torch.utils.data.DataLoader(train_dataset,\n                                               batch_size=cfg[\"batch_size\"],\n                                               shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset,\n                                               batch_size=cfg[\"batch_size\"],\n                                               shuffle=True)\n\n    # Start\n    for epoch in range(1, cfg[\"epoch_size\"]+1):\n        # Training\n        sum_loss = 0.0\n        sum_correct = 0\n        for imgs, true_labels in tqdm(train_loader):\n            network.zero_grad()\n            pred_probs = network(imgs)\n            pred_labels = torch.argmax(pred_probs, dim=1)\n            loss = criterion(pred_probs, true_labels)\n            loss.backward()\n            optimizer.step()\n            sum_loss += loss.item() * imgs.shape[0]\n            sum_correct += int(torch.sum(pred_labels == true_labels))\n        ave_loss = sum_loss / len(train_dataset)\n        accuracy = 100.0 * sum_correct / len(train_dataset)\n        result[\"Epoch\"].append(epoch)\n        result[\"Type\"].append(\"Training\")\n        result[\"Average Loss\"].append(ave_loss)\n        result[\"Accuracy\"].append(accuracy)\n        args = (epoch, cfg[\"epoch_size\"], ave_loss, accuracy)\n        print_str = \"[Training]Epoch:%d/%d,Average Loss:%.3f,Accuracy:%.2f%%\"\n        print(print_str % args)\n\n        # Validation\n        sum_loss = 0.0\n        sum_correct = 0\n        for imgs, true_labels in tqdm(valid_loader):\n            pred_probs = network(imgs)\n            pred_labels = torch.argmax(pred_probs, dim=1)\n            loss = criterion(pred_probs, true_labels)\n            sum_loss += loss.item() * imgs.shape[0]\n            sum_correct += int(torch.sum(pred_labels == true_labels))\n        ave_loss = sum_loss / len(valid_dataset)\n        accuracy = 100.0 * sum_correct / len(valid_dataset)\n        result[\"Epoch\"].append(epoch)\n        result[\"Type\"].append(\"Validation\")\n        result[\"Average Loss\"].append(ave_loss)\n        result[\"Accuracy\"].append(accuracy)\n        args = (epoch, cfg[\"epoch_size\"], ave_loss, accuracy)\n        print_str = \"[Validation]Epoch:%d/%d,Average Loss:%.3f,Accuracy:%.2f%%\"\n        print(print_str % args)\n\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nresult = learn(network,\n               train_dataset,\n               valid_dataset,\n               cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sns.relplot(x=\"Epoch\",\n            y=\"Average Loss\",\n            hue=\"Type\",\n            kind=\"line\",\n            data=pd.DataFrame(result))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sns.relplot(x=\"Epoch\",\n            y=\"Accuracy\",\n            hue=\"Type\",\n            kind=\"line\",\n            data=pd.DataFrame(result))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Investigation"},{"metadata":{},"cell_type":"markdown","source":"We execute the investigation about `Dig-MNIST.csv` by using trained network."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def invest(inv_dataset: KannadaMNISTDataset,\n           network: torch.nn.Module,\n           cfg: dict):\n    inv_true_labels = np.array([])\n    inv_pred_labels = np.array([])\n    inv_loader = torch.utils.data.DataLoader(inv_dataset,\n                                             batch_size=cfg[\"batch_size\"])\n\n    # Prediction\n    for imgs, true_labels in tqdm(inv_loader):\n        pred_probs = network(imgs)\n        pred_labels = torch.argmax(pred_probs, dim=1)\n        inv_true_labels = np.concatenate([inv_true_labels,\n                                          true_labels.cpu().numpy()])\n        inv_pred_labels = np.concatenate([inv_pred_labels,\n                                          pred_labels.cpu().numpy()])\n    return inv_true_labels, inv_pred_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\ninv_true_labels, inv_pred_labels = invest(inv_dataset, network, cfg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We overview results of the investigation."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"target_str = [\"Image No.%d\" % num for num in range(10)]\nreport_str = classification_report(inv_true_labels,\n                                   inv_pred_labels,\n                                   target_names=target_str,\n                                   digits=3)\nprint(report_str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"cm = pd.DataFrame(confusion_matrix(inv_true_labels, inv_pred_labels),\n                  columns=np.unique(inv_true_labels),\n                  index=np.unique(inv_pred_labels))\ncm.index.name = \"True Image No.\"\ncm.columns.name = \"Predicted Image No.\"\nsns.heatmap(cm, annot=True, cmap=\"Blues\", fmt=\"d\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We summarize by a confusion matrix and report as follows.\n* Weights of all image numbers are same because of `support`, thus the macro average and the weighted average are same.\n* Focusing on lower `precision` and higher `recall` than any other image numbers, `ThisNetwork` is tend to predict as `Image No.5` whatever true image number is.\n* Focussing on `f1-score`, `ThisNetwork` prediction is suitable for `Image No.2` and `Image No.5`, while not for `Image No.0` and `Image No.6`.\n    - We have to pay attention to observe the data difference between Learning Datasets and Investigation ones at `Image No.0` and `Image No.6`.\n* Focusing on the confusion matrix, `ThisNetwork` is tend to predict from true `Image No.7` as `Image No.6` and from true `Image No.1` as `Image No.0`.\n    - We have to pay attention to observe the image number difference at Investigation Datasets between true `Image No.7` and predicted `Image No.6`, and between `Image No.1` and predicted `Image No.0`."},{"metadata":{},"cell_type":"markdown","source":"## Observe Data Difference"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def show_30_imgs(dataset: KannadaMNISTDataset,\n                 label: int,\n                 title: str):\n    # Mask Images\n    mask = (dataset.labels == label).cpu()\n    imgs = dataset[mask][0].cpu()\n\n    # Show Top 30 Masked Images\n    fig, ax = plt.subplots(5, 6, sharex=True, sharey=True)\n    fig.suptitle(title)\n    for row in range(5):\n        for col in range(6):\n            idx = 6 * row + col\n            ax[row][col].set_xticklabels([]) \n            ax[row][col].set_yticklabels([]) \n            ax[row][col].imshow(imgs[idx][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(lrn_dataset, 0, \"[Learning] True Image No.0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(inv_dataset, 0, \"[Investigation] True Image No.0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(lrn_dataset, 6, \"[Learning] True Image No.6\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(inv_dataset, 6, \"[Investigation] True Image No.6\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We summarize as follows.\n* the `Image No.0` form in training dataset seems to be vertical form, but in investigation dataset not.\n    - We have to tune `scale` and `ratio` more widely in `torchvision.transforms.RandomResizedCrop`.\n* The form of `Image No.6` in learning dataset seems to be unified, while in investigation dataset not.\n* Almost all the `Image No.6` form look like epsilon letter, but some in investigation dataset is broken-formed epsilon letter.\n    - We have to add more noise to images in training datasets."},{"metadata":{},"cell_type":"markdown","source":"## Observe Image Number Difference"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def show_inv_error_30_imgs(inv_dataset: KannadaMNISTDataset,\n                           true_labels: np.ndarray,\n                           pred_labels: np.ndarray,\n                           true_num: int,\n                           pred_num: int):\n    # Mask Images\n    mask = (true_labels == true_num)\n    mask *= (pred_labels == pred_num)\n    err_pred_labels = pred_labels[mask]\n    err_true_labels = true_labels[mask]\n    err_imgs = inv_dataset[mask][0].cpu()\n\n    # Show Top 30 Masked Images\n    fig, ax = plt.subplots(5, 6, sharex=True, sharey=True)\n    args = (true_num, pred_num)\n    title = \"[Investigation] True Image No.%d, Predict Image No.%d\" % args\n    fig.suptitle(title)\n    for row in range(5):\n        for col in range(6):\n            idx = 6 * row + col\n            ax[row][col].set_xticklabels([]) \n            ax[row][col].set_yticklabels([]) \n            ax[row][col].imshow(err_imgs[idx][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(lrn_dataset, 7, \"[Learning] True Image No.7\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(lrn_dataset, 6, \"[Learning] True Image No.6\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_inv_error_30_imgs(inv_dataset,\n                       inv_true_labels,\n                       inv_pred_labels,\n                       7,\n                       6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(lrn_dataset, 1, \"[Learning] True Image No.1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_30_imgs(lrn_dataset, 0, \"[Learning] True Image No.0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_inv_error_30_imgs(inv_dataset,\n                       inv_true_labels,\n                       inv_pred_labels,\n                       1,\n                       0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We summarize as follows.\n* Like the previous summary, the form of `Image No.0`, `Image No.1`, `Image No.6` and `Image No.7` in learning dataset seems to be unified, while in investigation dataset complicated.\n* The `Image No.6` form in training dataset looks like epsilon letter and `Image No.7` like two letter, but some `Image No.7` in investigation dataset is the mixed form of epsilon and two letters.\n    - Huuum, `Image No.7` of investigation dataset seems to be confused images... We may seem to dismiss them.\n* In learning dataset, all `Image No.0` form seems to be connected and all `Image No.1` form seems not to be connected. But in investigation dataset, some `Image No.1` form seems not to be connected and to be rotated.\n    - We have to add more noise to images in training datasets.\n    - We have to tune `degree` more widely in `torchvision.transforms.RandomRotation`."},{"metadata":{},"cell_type":"markdown","source":"# A. Submitting"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def test(network: torch.nn.Module,\n         cfg: dict):\n    labels = []\n    df_csv = pd.read_csv(cfg[\"path_test_csv\"])\n    ids = df_csv[\"id\"]\n    imgs = df_csv.drop([\"id\"], axis=1).values.astype(np.int32)\n    # Reshape Image from (data_size, 784) to (data_size, 1, 1, 28, 28)\n    # Where Batch Size is 1\n    imgs = imgs.reshape(-1, 1, 1, 28, 28)\n\n    # Prediction\n    for id, img in zip(tqdm(ids), imgs):\n        # Scale Image from [0, 255] to [0.0, 1.0]\n        img = torch.tensor(img/255.0,\n                           dtype=torch.float32,\n                           device=cfg[\"device\"])\n        pred_probs = network(img)\n        pred_labels = torch.argmax(pred_probs, dim=1)\n        labels.append(pred_labels.cpu().numpy()[0])\n\n    result = pd.DataFrame({\"id\" : ids,\n                           \"label\" : labels})\n    result.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\ntest(network, cfg)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}