{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nimport torchvision\nimport os\nimport sys\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport math\nimport torchvision.transforms.functional as F\nimport torchvision.transforms as transforms\nimport time\nfrom sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score, precision_score, recall_score\nfrom PIL import Image\n\nCLASS_NAMES = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')\n\nNORM_MEAN = [0.485, 0.456, 0.406]\nNORM_STD = [0.229, 0.224, 0.225]\n\nMETRICS = {\n    'accuracy': {\n        'f': accuracy_score,\n        'args': {}\n    },\n    'balanced_accuracy': {\n        'f': balanced_accuracy_score,\n        'args': {}\n    },\n    'f1': {\n        'f': f1_score,\n        'args': {'average': 'weighted'}\n    },\n    'precision': {\n        'f': precision_score,\n        'args': {'average': 'weighted'}\n    },\n    'recall': {\n        'f': recall_score,\n        'args': {'average': 'weighted'}\n    }\n}\n\ndata_root = '/kaggle/input/digit-recognizer'\ntrain_file_name = 'train.csv'\ntest_file_name = 'test.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Figure\ndef make_image_label_figure(images, labels=None, class_names=None):\n    channels = images.shape[1]\n    if channels not in (3, 1):\n        raise ValueError(\"Images must have 1 or 3 channels\")\n    mean = NORM_MEAN if channels == 3 else [sum(NORM_MEAN) / 3]\n    std = NORM_STD if channels == 3 else [sum(NORM_STD) / 3]\n    mean = torch.tensor(mean)\n    std = torch.tensor(std)\n    mean = (-mean / std).tolist()\n    std = (1.0 / std).tolist()\n    n = int(math.sqrt(len(images)))\n    figure = plt.figure(figsize=(n, n))\n    figure.subplots_adjust(hspace=0.4, wspace=0.4)\n    for i in range(n*n):\n        image, label = images[i], (labels[i] if labels is not None else None)\n        image = F.normalize(image, mean=mean, std=std)\n        image = image.permute(1, 2, 0)\n        image = torch.squeeze(image)\n        image = (image * 255).int()\n        plt.subplot(n, n, i + 1, title=class_names[label] if label is not None else 'NA')\n        plt.xticks([])\n        plt.yticks([])\n        plt.grid(False)\n        plt.imshow(image, cmap='gray' if channels == 1 else None)\n    return figure","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Trainer\nclass Timer(object):\n\n    def __init__(self, v='ms', fmt='.3f'):\n        self.start = 0\n        self.val = 0\n        self.v = v\n        if v == 'ms':\n            self.d = 1000.0\n        elif v == 'mcs':\n            self.d = 1000000.0\n        elif v == 'ns':\n            self.d = 1000000000.0\n        else:\n            self.v = 's'\n            self.d = 1.0\n        self.fmt = fmt\n\n    def tick(self):\n        self.start = time.time()\n\n    def tock(self):\n        self.val = (time.time() - self.start) * self.d\n        return self.val\n\n    def __str__(self):\n        fmtstr = ('{0:' + self.fmt + '} ' + self.v).format(self.val)\n        return fmtstr\n\n\nclass ProgressBar(object):\n\n    def __init__(self, num):\n        self.num = num\n        self.done = 0\n        self.percent = 0.0\n\n    def update(self, percent):\n        self.percent = percent if percent <= 1.0 else 1.0\n        self.done = round(self.num * self.percent)\n\n    def __str__(self):\n        fmtstr = '[' + '=' * self.done + ' ' * (self.num - self.done) + '] ' + f\"{self.percent * 100.0:6.2f}%\"\n        return fmtstr\n\n\nclass AverageMeter(object):\n\n    def __init__(self, fmt='.3f'):\n        self.fmt = fmt\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.cnt = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.cnt = 0\n\n    def update(self, val):\n        self.val = val\n        self.sum += val\n        self.cnt += 1\n        self.avg = self.sum / self.cnt\n\n    def __str__(self):\n        fmtstr = ('{0:' + self.fmt + '}').format(self.avg)\n        return fmtstr\n\n\nclass AvgTimer(Timer):\n\n    def __init__(self, v='ms', fmt='.3f'):\n        super(AvgTimer, self).__init__(v=v, fmt=fmt)\n        self.avg_meter = AverageMeter(fmt=fmt)\n\n    def tock(self):\n        val = super(AvgTimer, self).tock()\n        self.avg_meter.update(val)\n        return val\n\n    def reset(self):\n        self.avg_meter.reset()\n\n    def __str__(self):\n        fmtstr = '{0}/{1} {2}'.format(super(AvgTimer, self).__str__(), str(self.avg_meter), self.v)\n        return fmtstr\n\n\nclass PyTorchTrainer(object):\n\n    def __init__(self, device=None, metrics=None):\n        self.device = torch.device(device) if device else torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n        self.num_epochs = 10\n        self.train_batches = 1\n        self.val_batches = 1\n        self.metrics = metrics or METRICS\n        self.batch_timer = AvgTimer(v='ms', fmt='.3f')\n        self.epoch_timer = AvgTimer(v='s', fmt='.3f')\n        self.train_timer = AvgTimer(v='s', fmt='.3f')\n        self.epoch_progress_bar = ProgressBar(10)\n\n    def train(self, model, optimizer, loss_criterion, train_dataloader, val_dataloader, scheduler=None, num_epochs=10):\n        # Variables\n        self.num_epochs = num_epochs\n        self.train_batches = len(train_dataloader)\n        self.val_batches = len(val_dataloader)\n        # Print log\n        self.log_start_train()\n        # Timings\n        self.train_timer.tick()\n        self.epoch_timer.reset()\n        for epoch in range(num_epochs):\n            # Timings\n            self.epoch_timer.tick()\n            # Print log\n            self.log_start_epoch(epoch)\n            # Train batch\n            train_loss, predictions, targets = self.forward_batches(model, optimizer, loss_criterion, train_dataloader,\n                                                                    epoch, train=True)\n            print()\n            # Val batches\n            val_loss, predictions, targets = self.forward_batches(model, optimizer, loss_criterion, val_dataloader,\n                                                                  epoch, train=False)\n            # Timings\n            self.epoch_timer.tock()\n            # Print log\n            self.log_end_epoch(train_loss, val_loss, predictions.cpu().numpy(), targets.cpu().numpy(), epoch)\n            # Make scheduler step\n            if scheduler:\n                scheduler.step(epoch=epoch)\n        # Timings\n        self.train_timer.tock()\n        # Print log\n        self.log_end_train()\n\n    def forward_batches(self, model, optimizer, loss_criterion, data_loader, epoch, train=True):\n        # Set model train or eval due to current phase\n        if train:\n            model.train()\n        else:\n            model.eval()\n        # Preset variables\n        avg_loss_value = AverageMeter(fmt='.3f')\n        all_predictions = None\n        all_targets = None\n        batches = self.train_batches if train else self.val_batches\n        # Timings\n        self.batch_timer.reset()\n        for batch_i, data in enumerate(data_loader, 1):\n            # Timings\n            self.batch_timer.tick()\n            # Forward batch\n            if train:\n                loss_value = self.forward_batch(model, optimizer, loss_criterion, data, train=train)\n            else:\n                loss_value, predictions, targets = self.forward_batch(model, optimizer, loss_criterion, data, train=train)\n                all_predictions = torch.cat((all_predictions, predictions)) if all_predictions is not None else predictions\n                all_targets = torch.cat((all_targets, targets)) if all_targets is not None else targets\n            avg_loss_value.update(loss_value)\n            # Timings\n            self.batch_timer.tock()\n            # Print log\n            self.log_batch(loss_value, batch_i, batches, epoch, train)\n        # Return\n        return avg_loss_value.avg, all_predictions, all_targets\n\n    def forward_batch(self, model, optimizer, loss_criterion, batch_data, train=True):\n        # Get Inputs and Targets and put them to device\n        inputs, targets = batch_data\n        inputs = inputs.to(self.device)\n        targets = targets.to(self.device)\n        with torch.set_grad_enabled(train):\n            # Forward model to get outputs\n            outputs = model.forward(inputs)\n            # Calculate Loss Criterion\n            loss = loss_criterion(outputs, targets)\n        if train:\n            # Zero optimizer gradients\n            optimizer.zero_grad()\n            # Calculate new gradients\n            loss.backward()\n            # Make optimizer step\n            optimizer.step()\n        # Variables\n        loss_value = loss.item()\n        if train:\n            return loss_value\n        predictions = outputs.argmax(dim=1).data\n        targets = targets.data\n        return loss_value, predictions, targets\n\n    def log_start_train(self):\n        print(f\"================================== Train ==================================\")\n        print(f\"== Number of epochs:           {self.num_epochs:6d}\")\n        print(f\"== Number of train batches:    {self.train_batches:6d}\")\n        print(f\"== Number of validate batches: {self.val_batches:6d}\")\n        print(f\"===========================================================================\")\n\n    def log_start_epoch(self, epoch):\n        print(f\"======== Epoch {epoch + 1}/{self.num_epochs}\")\n\n    def log_batch(self, loss, batch, batches, epoch, train):\n        self.epoch_progress_bar.update(batch / batches)\n        s = f\"\\r==== {'Train' if train else '  Val'} Batch {batch:6d}/{batches} {str(self.epoch_progress_bar)} \" \\\n            f\"loss[{loss:6.3f}] time[{str(self.batch_timer)}]\"\n        sys.stdout.write(s)\n\n    def log_end_epoch(self, train_loss, val_loss, predictions, targets, epoch):\n        s = f\"\\n======== train_loss[{train_loss:6.3f}] val_loss[{val_loss:6.3f}] \"\n        calculated_metrics = {}\n        for metric in self.metrics:\n            calculated_metrics[metric] = self.metrics[metric]['f'](predictions, targets, **self.metrics[metric]['args'])\n            s += f\"{metric}[{calculated_metrics[metric]:6.3f}] \"\n        s += f\"time[{str(self.epoch_timer)}]\\n\"\n        print(s)\n\n    def log_end_train(self):\n        print(f\"===========================================================================\")\n        print(f\"== time[{str(self.train_timer)}]\")\n        print(f\"================================== Train ==================================\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transforms\nclass Transforms(transforms.Compose):\n\n    def __init__(self, in_channels=1, out_channels=1, size=(32, 32), train=False):\n        if out_channels not in (3, 1) or in_channels not in (3, 1):\n            raise ValueError(\"Images must have 1 or 3 channels\")\n        mean = NORM_MEAN if out_channels == 3 else [sum(NORM_MEAN) / 3]\n        std = NORM_STD if out_channels == 3 else [sum(NORM_STD) / 3]\n        transforms_list = []\n        if in_channels != out_channels:\n            transforms_list.append(transforms.Grayscale(out_channels))\n        if train:\n            transforms_list.append(transforms.RandomAffine(15.0, translate=(0.1, 0.1), shear=(-0.25, 0.25, -0.25, 0.25)))\n        transforms_list.extend([\n            transforms.Resize(size),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=mean, std=std)\n        ])\n        super(Transforms, self).__init__(transforms_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset\nclass MNISTDataset(torch.utils.data.Dataset):\n    \n    def __init__(self, images, targets=None, transform=None):\n        super(MNISTDataset, self).__init__()\n        self.images = [Image.fromarray(image) for image in images]\n        self.targets = np.zeros(len(images)) if targets is None else targets.astype(int)\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.images)\n        \n    def __getitem__(self, index):\n        return self.images[index] if self.transform is None else self.transform(self.images[index]), torch.tensor(self.targets[index], dtype=torch.long)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read *.csv files to Pandas Dataframes\ntrain_df = pd.read_csv(os.path.join(data_root, train_file_name))\ntest_df = pd.read_csv(os.path.join(data_root, test_file_name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Construct from DataFrames train/val/test datasets\ntrain_targets = train_df.label.values.astype(int)\ntrain_images = (train_df.drop('label', axis=1).values.astype(float) / 255).reshape(-1, 28, 28)\ntest_images = (test_df.values.astype(float) / 255).reshape(-1, 28, 28)\nprint(\"Origin trainset images/targets shapes\", train_images.shape, train_targets.shape)\nprint(\"Origin testset images shape\", test_images.shape)\nprint()\nimages_train, images_val, targets_train, targets_val = train_test_split(train_images, train_targets, test_size=0.2)\nprint(\"Train images/targets shapes\", images_train.shape, targets_train.shape)\nprint(\"Val images/targets shapes\", images_val.shape, targets_val.shape)\nprint()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train/val/test dataloaders and datasets\n# Batch size\nbatch_size = 64\n# Datasets\ntrain_dataset = MNISTDataset(images_train, targets_train, transform=Transforms(in_channels=1, out_channels=1, size=(32, 32), train=True))\nval_dataset = MNISTDataset(images_val, targets_val, transform=Transforms(in_channels=1, out_channels=1, size=(32, 32), train=False))\ntest_dataset = MNISTDataset(test_images, transform=Transforms(in_channels=1, out_channels=1, size=(32, 32), train=False))\n# Dataloaders\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\nval_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\ntest_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show train batch\nimages, targets = next(iter(train_dataloader))\nfig = make_image_label_figure(images, targets, CLASS_NAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show val batch\nimages, targets = next(iter(val_dataloader))\nfig = make_image_label_figure(images, targets, CLASS_NAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show test batch\nimages = next(iter(test_dataloader))\nfig = make_image_label_figure(images[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MNISTNet(torch.nn.Module):\n    \n    def __init__(self, in_channels=1, classes=10):\n        super(MNISTNet, self).__init__()\n        self.in_channels = in_channels\n        self.classes = classes\n        \n        self.feature_extractor = self.make_feature_extractor()  # 128x2x2\n        self.pool = torch.nn.AdaptiveAvgPool2d((1, 1))  # 128x1x1\n        self.classifier = self.make_classifier()  # 10\n        \n        self.sm = torch.nn.Softmax()\n        \n        # weight initialization\n        for m in self.modules():\n            if isinstance(m, torch.nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight, mode='fan_out')\n                if m.bias is not None:\n                    torch.nn.init.zeros_(m.bias)\n            elif isinstance(m, torch.nn.BatchNorm2d):\n                torch.nn.init.ones_(m.weight)\n                torch.nn.init.zeros_(m.bias)\n            elif isinstance(m, torch.nn.Linear):\n                torch.nn.init.normal_(m.weight, 0, 0.01)\n                torch.nn.init.zeros_(m.bias)\n    \n    def forward(self, x):\n        x = self.feature_extractor(x)\n        x = self.pool(x)\n        x = x.view(-1, 1 * 1 * 128)\n        x = self.classifier(x)\n        return x\n    \n    def predict(self, x):\n        x = self.forward(x)\n        x = self.sm(x)\n        return x\n    \n    def make_feature_extractor(self):\n        layers = [\n            torch.nn.Conv2d(self.in_channels, 16, kernel_size=3, stride=2, padding=1),  # 16x16x16\n            torch.nn.BatchNorm2d(16),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1),  # 16x16x16\n            torch.nn.BatchNorm2d(16),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=1),  # 32x8x8\n            torch.nn.BatchNorm2d(32),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),  # 32x8x8\n            torch.nn.BatchNorm2d(32),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1),  # 64x4x4\n            torch.nn.BatchNorm2d(64),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),  # 64x4x4\n            torch.nn.BatchNorm2d(64),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1),  # 128x2x2\n            torch.nn.BatchNorm2d(128),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),  # 128x2x2\n            torch.nn.BatchNorm2d(128),\n            torch.nn.ReLU(inplace=True)\n        ]\n        return torch.nn.Sequential(*layers)\n    \n    def make_classifier(self):\n        layers = [\n            torch.nn.Dropout(0.2),\n            torch.nn.Linear(1 * 1 * 128, 84),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Dropout(0.2),\n            torch.nn.Linear(84, 32),\n            torch.nn.ReLU(inplace=True),\n            torch.nn.Dropout(0.2),\n            torch.nn.Linear(32, self.classes)\n        ]\n        return torch.nn.Sequential(*layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n\nnet = MNISTNet(1, 10)\nprint(net)\nnet = net.to(torch.device(device))\n\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(net.parameters(), lr=0.001, weight_decay=0.00004)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)\n\ntrainer = PyTorchTrainer(device=device)\ntrainer.train(net, optimizer, criterion, train_dataloader, val_dataloader, scheduler=scheduler, num_epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set model to evaluation mode\nnet.eval()\nfor param in net.parameters():\n    param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict test images\nall_predictions = None\nfor batch in test_dataloader:\n    inputs = batch[0]\n    inputs = inputs.to(torch.device(device))\n    outputs = net.forward(inputs)\n    predictions = outputs.argmax(dim=1).data\n    all_predictions = torch.cat((all_predictions, predictions)) if all_predictions is not None else predictions\nall_predictions = all_predictions.cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show val batch with predictions\nimages = next(iter(val_dataloader))[0]\ncuda_images = images.to(torch.device(device))\npredictions = net.forward(cuda_images).argmax(dim=1).data.cpu()\nfig = make_image_label_figure(images, predictions, CLASS_NAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show test batch with predictions\nimages = next(iter(test_dataloader))[0]\ncuda_images = images.to(torch.device(device))\npredictions = net.forward(cuda_images).argmax(dim=1).data.cpu()\nfig = make_image_label_figure(images, predictions, CLASS_NAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame(np.c_[np.arange(1, len(all_predictions)+1)[:,None], all_predictions], columns=['ImageId', 'Label'])\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)\n!cat submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}