{"cells":[{"metadata":{"id":"no8KTYF__vJ_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import csv\nimport math\nimport numpy as np\nimport os\nimport pandas as pd\n\nimport torch\nfrom torch import nn\nfrom torch.optim.optimizer import Optimizer, required\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms as T, utils, models, datasets","execution_count":null,"outputs":[]},{"metadata":{"id":"I1jXo2tJENVu","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class Config:\n    split_dataset = True\n    valid_frac = 0.1\n    \n    model_path = '/kaggle/saves/kannada.pth'\n    train_valid_csv = \"/kaggle/input/Kannada-MNIST/train.csv\"\n    train_csv = \"/kaggle/saves/train_.csv\"\n    valid_csv = \"/kaggle/saves/valid_.csv\"\n    test_csv = \"/kaggle/input/Kannada-MNIST/test.csv\"\n    output_csv = \"submission.csv\"\n\n    train_transforms = T.Compose([\n        T.ToPILImage(),\n        T.RandomPerspective(),\n        T.RandomRotation(15),\n        T.RandomResizedCrop(28, scale=(0.8, 1.2), ratio=(0.75, 1.33), interpolation=2),\n        T.RandomHorizontalFlip(p=0.5),\n        T.ToTensor(),\n        T.Normalize((0.1307,), (0.3081,))\n    ])\n\n    valid_transforms = T.Compose([\n        T.ToTensor(),\n        T.Normalize((0.1307,), (0.3081,))\n    ])\n\n    device = torch.device(\"cuda:0\")\n    model = models.mobilenet_v2\n#     print(model)\n    criterion = nn.CrossEntropyLoss()\n    epochs = 30\n    batch_size = 32\n\n    log_interval = 337\n    min_loss = 10**9\n","execution_count":null,"outputs":[]},{"metadata":{"id":"YTdeRd15GhT7","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"if Config.split_dataset:\n    if not os.path.exists(\"/kaggle/saves\"):\n        os.makedirs(\"/kaggle/saves\")\n    \n    train_valid = pd.read_csv(Config.train_valid_csv)\n    valid = train_valid.sample(frac=Config.valid_frac)\n    train = train_valid.drop(valid.index)\n    valid.to_csv(Config.valid_csv, index=False)\n    train.to_csv(Config.train_csv, index=False)\n    \n    for root, dirs, files in os.walk(\"/kaggle\"):\n        print(root, dirs, files)","execution_count":null,"outputs":[]},{"metadata":{"id":"b7lKJlyaRDv6","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, degenerated_to_sgd=True):\n        if not 0.0 <= lr:\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n        if not 0.0 <= eps:\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\n        \n        self.degenerated_to_sgd = degenerated_to_sgd\n        if isinstance(params, (list, tuple)) and len(params) > 0 and isinstance(params[0], dict):\n            for param in params:\n                if 'betas' in param and (param['betas'][0] != betas[0] or param['betas'][1] != betas[1]):\n                    param['buffer'] = [[None, None, None] for _ in range(10)]\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, buffer=[[None, None, None] for _ in range(10)])\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = group['buffer'][int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    elif self.degenerated_to_sgd:\n                        step_size = 1.0 / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = -1\n                    buffered[2] = step_size\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:\n                    if group['weight_decay'] != 0:\n                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)\n                    p.data.copy_(p_data_fp32)\n                elif step_size > 0:\n                    if group['weight_decay'] != 0:\n                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n                    p_data_fp32.add_(-step_size * group['lr'], exp_avg)\n                    p.data.copy_(p_data_fp32)\n\n        return loss\n","execution_count":null,"outputs":[]},{"metadata":{"id":"MoL162aMBVu_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class CsvImages(Dataset):\n    def __init__(self, csv_file, has_labels=True, transform=None):\n        self.df = pd.read_csv(csv_file)\n        self.has_labels = has_labels\n        self.transform = transform\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, i):\n        sample = self.df.loc[i]\n        if self.has_labels:\n            label = sample.pop('label')\n        else:\n            id = sample.pop('id')\n        sample = sample.to_numpy(np.uint8).reshape(28, 28, 1)\n\n        if self.transform is not None:\n            sample = self.transform(sample)\n\n        if self.has_labels:\n            return sample, label\n        else:\n            return sample, id","execution_count":null,"outputs":[]},{"metadata":{"id":"opOl904bNKW2","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def save(model, valid_loss):\n    print(f'Saving model with best validation loss = {valid_loss:.7f} at {Config.model_path}\\n')\n    torch.save(model.state_dict(), Config.model_path)\n    \ndef load(model):\n    print(f'Loading best version of the model, with validation loss = {Config.min_loss:.7f}\\n')\n    model.load_state_dict(torch.load(Config.model_path))\n    model.eval()\n\ndef train(model, criterion, device, train_loader, optimizer, epoch):\n    model.train()\n    sum_loss = 0\n    correct = 0\n    num_batches = len(train_loader)\n    print(f'\\n\\nTrain Epoch: {epoch}\\n')\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.to(device), target.to(device)\n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        sum_loss += loss.item()\n        pred = output.argmax(dim=1, keepdim=True)\n        correct += pred.eq(target.view_as(pred)).sum().item()\n        loss.backward()\n        optimizer.step()\n        if batch_idx % Config.log_interval == 0 and batch_idx != 0:\n            progress = 100. * (batch_idx+1) // len(train_loader)\n            num_examples = (batch_idx+1) * Config.batch_size\n            avg_acc = 100. * correct / num_examples\n            avg_loss = sum_loss / (batch_idx+1)\n            print(f'[{batch_idx}/{num_batches} ({progress}%)]\\tAverage loss: {avg_loss:.7f}\\tAccuracy: {correct}/{num_examples} ({avg_acc:.2f}%)\\n', end='\\r')\n\n\ndef valid(model, criterion, device, valid_loader):\n    model.eval()\n    valid_loss = 0\n    correct = 0\n    with torch.no_grad():\n        for data, target in valid_loader:\n            data, target = data.to(device), target.to(device)\n            output = model(data)\n            valid_loss += criterion(output, target).item()\n            pred = output.argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    valid_loss /= len(valid_loader.dataset)\n\n    val_acc = 100. * correct / len(valid_loader.dataset)\n    print(f'\\nValid set: Average loss: {valid_loss:.7f}, Accuracy: {correct}/{len(valid_loader.dataset)} ({val_acc:.2f}%)\\n')\n    if valid_loss < Config.min_loss:\n        Config.min_loss = valid_loss\n        save(model, valid_loss)\n    \ndef test(model, device, test_loader, output_csv):\n    model.eval()\n    with open(output_csv, mode='w') as out_csv:\n        fieldnames = ['id', 'label']\n        writer = csv.DictWriter(out_csv, fieldnames=fieldnames)\n        writer.writeheader()\n\n        with torch.no_grad():\n            for data, ids in test_loader:\n                data = data.to(device)\n                output = model(data)\n                preds = output.argmax(dim=1).detach().cpu().numpy()\n                for id, pred in zip(ids, preds):\n                    writer.writerow({'id': id.item(), 'label': pred})\n\n    print(f'\\n Finished predicting labels on test dataset. Wrote output csv at {Config.output_csv}')","execution_count":null,"outputs":[]},{"metadata":{"id":"ill8S5zWEbcX","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train_set = CsvImages(Config.train_csv, transform=Config.train_transforms)\ntrain_loader = DataLoader(train_set, batch_size=Config.batch_size,\n                          shuffle=True, num_workers=4)\n\nvalid_set = CsvImages(Config.valid_csv, transform=Config.valid_transforms)\nvalid_loader = DataLoader(valid_set, batch_size=Config.batch_size,\n                          shuffle=True, num_workers=4)\n\ntest_set = CsvImages(Config.test_csv, has_labels=False, transform=Config.valid_transforms)\ntest_loader = DataLoader(test_set, batch_size=Config.batch_size,\n                         shuffle=True, num_workers=4)\n\nmodel = Config.model(pretrained=False, progress=True, num_classes=10)\nmodel.features[0][0] = nn.Conv2d(1, 32, kernel_size=3, stride=2, padding=1, bias=False)\nmodel.classifier = nn.Sequential(\n    nn.Dropout(p=0.2),\n    nn.Linear(1280, 256),\n    nn.Linear(256, 10)\n)\nmodel = model.to(Config.device)\noptimizer = RAdam(model.parameters())\nprint(model)","execution_count":null,"outputs":[]},{"metadata":{"id":"y9dvDBBrRaO5","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"383d10d9-d953-43ff-881e-9003f34d396c","trusted":true},"cell_type":"code","source":"for epoch in range(1, Config.epochs + 1):\n        train(model, Config.criterion, Config.device, train_loader, optimizer, epoch)\n        valid(model, Config.criterion, Config.device, valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load(model)\ntest(model, Config.device, test_loader, Config.output_csv)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}