{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install easydict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install pytorch-lightning-bolts","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2c1d0fc3-b908-4f63-b391-e03bfc75e75b","_cell_guid":"430b00cf-a61d-4255-92b6-52b291da0a58","trusted":true},"cell_type":"code","source":"from argparse import ArgumentParser\nimport pandas as pd\nimport numpy as np\nimport torch\nimport pytorch_lightning as pl\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader, random_split,Dataset\nimport pl_bolts\nfrom pl_bolts.callbacks.vision import ConfusedLogitCallback\nfrom torchvision.datasets.mnist import MNIST\nfrom torchvision import transforms,models\nimport easydict\n\n\nclass Backbone(torch.nn.Module):\n    def __init__(self, hidden_dim=128):\n        super().__init__()\n        self.l1 = torch.nn.Linear(28 * 28, hidden_dim)\n        self.l2 = torch.nn.Linear(hidden_dim, 10)\n\n    def forward(self, x):\n        x = x.view(x.size(0), -1)\n        x = torch.relu(self.l1(x))\n        x = torch.relu(self.l2(x))\n        return x\n\n\nclass LitClassifier(pl.LightningModule):\n    def __init__(self, backbone, learning_rate=1e-3):\n        super().__init__()\n        self.save_hyperparameters()\n        self.backbone = backbone\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self.backbone(x)\n        loss = F.cross_entropy(y_hat, y)\n        self.log('train_loss', loss, on_epoch=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self.backbone(x)\n        loss = F.cross_entropy(y_hat, y)\n        self.log('valid_loss', loss, on_step=True)\n\n    def test_step(self, batch, batch_idx):\n#         x= batch\n#         y_hat = self.backbone(x)\n#         loss = F.cross_entropy(y_hat, y)\n        #self.log('test_loss',)\n        x=batch\n        test_pred = torch.LongTensor()\n        y_hat = self.backbone(x)\n        pred = self.output.gpu().data.max(1, keepdim=True)[1]\n        test_pred = torch.cat((test_pred, pred), dim=0)\n        self.log('test_loss', test_pred)\n        return test_pred\n\n    def configure_optimizers(self):\n        # self.hparams available because we called self.save_hyperparameters()\n        return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate)\n\n    @staticmethod\n    def add_model_specific_args(parent_parser):\n        parser = ArgumentParser(parents=[parent_parser], add_help=False)\n        #parser.add_argument('--learning_rate', type=float, default=0.0001)\n        return parser\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MNISTDataset(Dataset):\n    \"\"\"MNIST dtaa set\"\"\"\n    \n    def __init__(self, dataframe, \n                 transform = transforms.Compose([transforms.ToPILImage(),\n                                                 transforms.ToTensor(),\n                                                 transforms.Normalize(mean=(0.5,), std=(0.5,))])\n                ):\n        df = dataframe\n        # for MNIST dataset n_pixels should be 784\n        self.n_pixels = 784\n        \n        if len(df.columns) == self.n_pixels:\n            # test data\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            # training data\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n            \n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        if self.y is not None:\n            return self.transform(self.X[idx]), self.y[idx]\n        else:\n            return self.transform(self.X[idx])\n\n\ndef get_dataset(dataframe, dataset=MNISTDataset,\n                transform=transforms.Compose([transforms.ToPILImage(),\n                                              transforms.ToTensor(),\n                                              transforms.Normalize(mean=(0.5,), std=(0.5,))])):\n    return dataset(dataframe, transform=transform)\n\ndef split_dataframe(dataframe=None, fraction=0.9):\n    df_1 = dataframe.sample(frac=fraction)\n    df_2 = dataframe.drop(df_1.index)\n    return df_1, df_2\n\ntrain_csv_path = '/kaggle/input/digit-recognizer/train.csv'\ntest_csv_path = '/kaggle/input/digit-recognizer/test.csv'\n\ntrain_df = pd.read_csv(train_csv_path)\ntest_df = pd.read_csv(test_csv_path)\n\n\nrandom_affine = transforms.RandomAffine(degrees=45, translate=(0.2, 0.2), scale=(0.7, 1.3))\nrandom_resized_crop =  transforms.RandomResizedCrop(28)\nrandom_horizontal_flip = transforms.RandomHorizontalFlip()\nrandom_perspective = transforms.RandomPerspective()\n\ntrain_transforms = transforms.Compose(\n    [transforms.ToPILImage(),\n     transforms.RandomChoice([random_affine,random_resized_crop,random_horizontal_flip]),\n     transforms.ToTensor(),\n     transforms.Normalize(mean=(0.5,), std=(0.5,)),\n    transforms.RandomErasing(),\n    ])\n\nval_test_transforms = transforms.Compose(\n    [transforms.ToPILImage(),\n     transforms.ToTensor(),\n     transforms.Normalize(mean=(0.5,), std=(0.5,))])\n\n\n#train_df_new, val_df = split_dataframe(dataframe=train_df, fraction=0.9)\n\nmnist_dataset = get_dataset(train_df, transform=train_transforms)\n#train_dataset = get_dataset(train_df_new, transform=train_transforms)\n#val_dataset = get_dataset(val_df, transform=val_test_transforms)\ntest_dataset = get_dataset(test_df, transform=val_test_transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cli_main():\n    pl.seed_everything(1234)\n\n    # ------------\n    # args\n    # ------------\n#     parser = ArgumentParser()\n#     parser.add_argument('--batch_size', default=32, type=int)\n#     parser.add_argument('--hidden_dim', type=int, default=128)\n#     parser = pl.Trainer.add_argparse_args(parser)\n#     parser = LitClassifier.add_model_specific_args(parser)\n#     args = parser.parse_args()\n    args = easydict.EasyDict({ \"batch_size\": 64, \"hidden_dim\": 128,\"gpus\":1,\"learning_late\":0.01,\"auto_lr_find\":True,\"max_epochs\":2})\n\n    # ------------\n    # data\n    # ------------\n#     dataset = MNIST('', train=True, download=True, transform=transforms.ToTensor())\n#     mnist_test = MNIST('', train=False, download=True, transform=transforms.ToTensor()),\n#     mnist_train, mnist_val = random_split(dataset, [55000, 5000])\n\n#     train_loader = DataLoader(mnist_train, batch_size=args.batch_size)\n#     val_loader = DataLoader(mnist_val, batch_size=args.batch_size)\n#     test_loader = DataLoader(mnist_test, batch_size=args.batch_size)\n    \n    mnist_train, mnist_val = random_split(mnist_dataset, [int(len(mnist_dataset)*0.9), int(len(mnist_dataset))-int(len(mnist_dataset)*0.9)])\n    \n    train_loader = torch.utils.data.DataLoader(dataset=mnist_train, batch_size=args.batch_size)\n    val_loader = torch.utils.data.DataLoader(dataset=mnist_val,batch_size=args.batch_size)        \n\n    test_loader = torch.utils.data.DataLoader(dataset=test_dataset,batch_size=args.batch_size)\n    # model\n    # ------------\n    model = LitClassifier(Backbone(hidden_dim=args.hidden_dim))\n\n    # ------------\n    # Callbacks\n    # ------------\n    early_stopping = pl.callbacks.EarlyStopping('valid_loss_epoch')\n    #confused_logit = ConfusedLogitCallback(top_k=10)\n    \n    # ------------\n    # training\n    # ------------\n    #trainer = pl.Trainer.from_argparse_args(args,callbacks=[early_stopping,confused_logit])\n    trainer = pl.Trainer.from_argparse_args(args,callbacks=[early_stopping])\n    trainer.fit(model, train_loader, val_loader)\n\n    # ------------\n    # testing\n    # ------------\n    result = trainer.test(test_dataloaders=test_loader)\n    print(result)\n\n\nif __name__ == '__main__':\n    cli_main()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ca2d26b-1bb0-4e41-a882-bb61cdcf03b7","_cell_guid":"8c553dac-97b1-485b-8276-99119592fd7e","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport plotly.express as px\n\nimport pytorch_lightning as pl\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import  \nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms, models\nfrom torchvision.utils import make_grid\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nimport matplotlib.pyplot as plt\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8421cf4c-18a6-4cb8-8cf4-816b4a0855e7","_cell_guid":"990f599d-f8f9-4ffc-a906-bd8190fa0732","trusted":true},"cell_type":"code","source":"train_csv_path = '/kaggle/input/digit-recognizer/train.csv'\ntest_csv_path = '/kaggle/input/digit-recognizer/test.csv'\n\ntrain_df = pd.read_csv(train_csv_path)\ntest_df = pd.read_csv(test_csv_path)\n\n# have a glimpse of train dataframe structure\nn_train = train_df.shape[0]\nn_pixels = train_df.shape[1] - 1\nn_class = len(set(train_df['label']))\nprint('Number of training samples: {0}'.format(n_train))\nprint('Number of training pixels: {0}'.format(n_pixels))\nprint('Number of classes: {0}'.format(n_class))\nprint(train_df.head())\n\n# have a glimpse of test dataframe structure\nn_test = test_df.shape[0]\nn_pixels = test_df.shape[1]\nprint('Number of test samples: {0}'.format(n_test))\nprint('Number of test pixels: {0}'.format(n_pixels))\nprint(test_df.head())\n\ntrain_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c53a26ce-ee12-4ab6-a359-25ed2b8dc86c","_cell_guid":"bdcb655f-244a-43bf-be13-cd57e05c953e","trusted":true},"cell_type":"code","source":"random_idx = np.random.randint(n_train, size=8)\n\ngrid = make_grid(torch.Tensor((train_df.iloc[random_idx, 1:].values/255.).reshape((-1, 28, 28))).unsqueeze(1), nrow=8)\nplt.rcParams['figure.figsize'] = (16, 2)\nplt.imshow(grid.numpy().transpose((1,2,0)))\nplt.axis('off')\nprint(*list(train_df.iloc[random_idx, 0].values), sep = ', ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv_path = '/kaggle/input/digit-recognizer/train.csv'\ntest_csv_path = '/kaggle/input/digit-recognizer/test.csv'\n\ntrain_df = pd.read_csv(train_csv_path)\ntest_df = pd.read_csv(test_csv_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08efd091-9677-431f-aac9-ff5c16183a71","_cell_guid":"dda3ed8d-3ba1-4d55-b25d-ec17cfd79fd2","trusted":true},"cell_type":"code","source":"class MNISTDataset(Dataset):\n    \"\"\"MNIST dtaa set\"\"\"\n    \n    def __init__(self, dataframe, \n                 transform = transforms.Compose([transforms.ToPILImage(),\n                                                 transforms.ToTensor(),\n                                                 transforms.Normalize(mean=(0.5,), std=(0.5,))])\n                ):\n        df = dataframe\n        # for MNIST dataset n_pixels should be 784\n        self.n_pixels = 784\n        \n        if len(df.columns) == self.n_pixels:\n            # test data\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            # training data\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n            \n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        if self.y is not None:\n            return self.transform(self.X[idx]), self.y[idx]\n        else:\n            return self.transform(self.X[idx])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d41b1358-8d89-4803-a547-c083bbec8dc2","_cell_guid":"d54b9d8d-3097-4e4f-bb08-ddbefd3f2d3c","trusted":true},"cell_type":"code","source":"random_affine = transforms.RandomAffine(degrees=45, translate=(0.2, 0.2), scale=(0.7, 1.3))\nrandom_resized_crop =  transforms.RandomResizedCrop(28)\nrandom_horizontal_flip = transforms.RandomHorizontalFlip()\nrandom_perspective = transforms.RandomPerspective()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8095773-5531-4170-b28d-90ae26cfbe63","_cell_guid":"d9dd01e4-6058-4e48-886b-def5cbe7842c","trusted":true},"cell_type":"code","source":"\ntrain_transforms = transforms.Compose(\n    [transforms.ToPILImage(),\n     transforms.RandomChoice([random_affine,random_resized_crop,random_horizontal_flip]),\n     transforms.ToTensor(),\n     transforms.Normalize(mean=(0.5,), std=(0.5,)),\n    transforms.RandomErasing(),\n    ])\n\nval_test_transforms = transforms.Compose(\n    [transforms.ToPILImage(),\n     transforms.ToTensor(),\n     transforms.Normalize(mean=(0.5,), std=(0.5,))])\n\ndef get_dataset(dataframe, dataset=MNISTDataset,\n                transform=transforms.Compose([transforms.ToPILImage(),\n                                              transforms.ToTensor(),\n                                              transforms.Normalize(mean=(0.5,), std=(0.5,))])):\n    return dataset(dataframe, transform=transform)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4de83f90-26c7-4967-a75c-c0acff69a712","_cell_guid":"a073621f-e56e-4944-9929-65d84352682b","trusted":true},"cell_type":"code","source":"def convert_relu_to_gelu(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, nn.GELU())\n        else:\n            convert_relu_to_gelu(child)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0051d41f-3e6c-4724-b5d9-3035ad2eb17c","_cell_guid":"895d4631-f872-4c33-b975-4698c091fc5b","trusted":true},"cell_type":"code","source":"from torchvision.models.resnet import ResNet, BasicBlock, Bottleneck\n\nclass MNISTResNet(ResNet):\n    def __init__(self):\n        super(MNISTResNet, self).__init__(BasicBlock, [2, 2, 2, 2], num_classes=10) # Based on ResNet18\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=1, padding=3,bias=False)\n\nmodel = MNISTResNet()\nconvert_relu_to_gelu(model)\nprint(model)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fbcd761c-5694-44ba-a489-f9b9a09428a9","_cell_guid":"97d178f0-4ba7-41df-80ae-14c14080416b","trusted":true},"cell_type":"code","source":"from torch.optim.optimizer import Optimizer, required\n\nclass DemonRanger(Optimizer):\n\n    # Rectified-AMSGrad/RAdam + AdaMod + QH Momentum + Iterat Averaging + Lookahead + DEMON (decaying Momentum) + gradient centralization + grad noise\n\n    def __init__(self, params, lr=1e-3,\n                 betas=(0.999, 0.999, 0.999),\n                 nus=(0.7, 1.0),\n                 eps=1e-8,\n                 k=5,\n                 alpha=0.8,\n                 gamma=0.55,\n                 use_demon=True,\n                 rectify=True,\n                 amsgrad=True,\n                 AdaMod=True,\n                 AdaMod_bias_correct=True,\n                 IA=True,\n                 IA_cycle=1000,\n                 epochs=100,\n                 step_per_epoch=None,\n                 weight_decay=0,\n                 use_gc=True,\n                 use_grad_noise=False,\n                 use_diffgrad=False,\n                 dropout=0.0):\n\n        # betas = (beta1 for first order moments, beta2 for second order moments, beta3 for ema over adaptive learning rates (AdaMod))\n        # nus = (nu1,nu2) (for quasi hyperbolic momentum)\n        # eps = small value for numerical stability (avoid divide by zero)\n        # k = lookahead cycle\n        # alpha = outer learning rate (lookahead)\n        # gamma = gradient noise control parameter (for regularization)\n        # use_demon = bool to decide whether to use DEMON (Decaying Momentum) or not\n        # rectify = bool to decide whether to apply the recitification term (from RAdam) or not\n        # amsgrad = bool to decide whether to use amsgrad instead of adam as the core optimizer\n        # AdaMod_bias_correct = bool to decide whether to add bias correction to AdaMod\n        # IA = bool to decide if Iterate Averaging is ever going to be used\n        # IA_cycle = Iterate Averaging Cycle (Recommended to initialize with no. of iterations in Epoch) (doesn't matter if you are not using IA)\n        # epochs = No. of epochs you plan to use (Only relevant if using DEMON)\n        # step_per_epoch = No. of iterations in an epoch (only relevant if using DEMON)\n        # weight decay = decorrelated weight decay value\n        # use_gc = bool to determine whether to use gradient centralization or not.\n        # use_grad_noise = bool to determine whether to use gradient noise or not.\n        # use_diffgrad = bool to determine whether to use diffgrad or not.\n        # dropout = learning rate dropout, probability of setting learning rate to zero\n\n        if not 0.0 <= lr:\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n        if not 0.0 <= eps:\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\n                \"Invalid beta parameter at index 0: {}\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\n                \"Invalid beta parameter at index 1: {}\".format(betas[1]))\n        if not 0.0 <= betas[2] < 1.0:\n            raise ValueError(\n                \"Invalid beta parameter at index 2: {}\".format(betas[2]))\n        if not 0.0 <= nus[0] <= 1.0:\n            raise ValueError(\n                \"Invalid nu parameter at index 0: {}\".format(nus[0]))\n        if not 0.0 <= nus[1] <= 1.0:\n            raise ValueError(\n                \"Invalid nu parameter at index 1: {}\".format(nus[1]))\n        if not 0.0 <= alpha <= 1.0:\n            raise ValueError(\"Invalid alpha parameter: {}\".format(alpha))\n        if not 0.0 <= dropout < 1.0:\n            raise ValueError(\"Invalid dropout parameter: {}\".format(dropout))\n\n        self.use_gc = use_gc\n        self.use_grad_noise = use_grad_noise\n        self.use_diffgrad = use_diffgrad\n        self.k = k\n        self.epochs = epochs\n        self.amsgrad = amsgrad\n        self.use_demon = use_demon\n        self.IA_cycle = IA_cycle\n        self.IA = IA\n        self.rectify = rectify\n        self.AdaMod = AdaMod\n        self.AdaMod_bias_correct = AdaMod_bias_correct\n        if step_per_epoch is None:\n            self.step_per_epoch = IA_cycle\n        else:\n            self.step_per_epoch = step_per_epoch\n\n        self.T = self.epochs * self.step_per_epoch\n\n        defaults = dict(lr=lr,\n                        betas=betas,\n                        nus=nus,\n                        eps=eps,\n                        alpha=alpha,\n                        gamma=gamma,\n                        weight_decay=weight_decay,\n                        dropout=dropout)\n        super(DemonRanger, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(DemonRanger, self).__setstate__(state)\n\n    def apply_AdaMod(self, beta3, n_avg, n, step):\n        n_avg.mul_(beta3).add_(1 - beta3, n)\n        if self.AdaMod_bias_correct:\n            n_avg_ = n_avg.clone()\n            n_avg_.div_(1 - (beta3 ** step))\n            torch.min(n, n_avg_, out=n)\n        else:\n            torch.min(n, n_avg, out=n)\n        return n\n\n    def step(self, activate_IA=False, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError(\n                        'DemonRanger does not support sparse gradients')\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p.data)\n                    state['exp_avg_sq'] = torch.zeros_like(p.data)\n                    if self.use_diffgrad:\n                        state['previous_grad'] = torch.zeros_like(p.data)\n                    state['num_models'] = 0\n                    state['cached_params'] = p.data.clone()\n                    if self.amsgrad:\n                        state['max_exp_avg_sq'] = torch.zeros_like(p.data)\n                    if self.AdaMod:\n                        state['n_avg'] = torch.zeros_like(p.data)\n\n                state['step'] += 1\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1_init, beta2, beta3 = group['betas']\n                rho_inf = (2 / (1 - beta2)) - 1\n                nu1, nu2 = group['nus']\n                lr = group['lr']\n                wd = group['weight_decay']\n                alpha = group['alpha']\n                gamma = group['gamma']\n\n                do_IA = False\n                lookahead_step = False\n\n                if self.IA and activate_IA:\n                    lookahead_step = False\n                    if state['step'] % self.IA_cycle == 0:\n                        do_IA = True\n                elif self.k == 0:\n                    lookahead_step = False\n                else:\n                    if state['step'] % self.k == 0:\n                        lookahead_step = True\n                    else:\n                        lookahead_step = False\n\n                if self.use_demon:\n                    temp = 1 - (state['step'] / self.T)\n                    beta1 = beta1_init * temp / \\\n                        ((1 - beta1_init) + beta1_init * temp)\n                else:\n                    beta1 = beta1_init\n\n                if self.use_grad_noise:\n                    grad_var = lr / ((1 + state['step'])**gamma)\n                    grad_noise = torch.empty_like(grad).normal_(\n                        mean=0.0, std=math.sqrt(grad_var))\n                    grad.add_(grad_noise)\n\n                if self.use_gc and grad.dim() > 1:\n                    grad.add_(-grad.mean(dim=tuple(range(1, grad.dim())), keepdim=True))\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                if self.use_diffgrad:\n                    previous_grad = state['previous_grad']\n                    diff = abs(previous_grad - grad)\n                    dfc = 1. / (1. + torch.exp(-diff))\n                    state['previous_grad'] = grad.clone()\n                    exp_avg = exp_avg * dfc\n\n                momentum = exp_avg.clone()\n                momentum.div_(\n                    1 - (beta1 ** state['step'])).mul_(nu1).add_(1 - nu1, grad)\n\n                if wd != 0:\n                    p.data.add_(-wd * lr, p.data)\n\n                beta2_t = beta2 ** state['step']\n\n                if self.amsgrad and state['step'] > 1:\n                    max_exp_avg_sq = state['max_exp_avg_sq']\n                    # Maintains the maximum of all 2nd moment running avg. till now\n                    torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)\n                    vt = max_exp_avg_sq.clone()\n                else:\n                    vt = exp_avg_sq.clone()\n\n                if self.rectify:\n                    rho_t = rho_inf - 2 * \\\n                        state['step'] * beta2_t / (1 - beta2_t)\n\n                    # more conservative since it's an approximated value\n                    if rho_t >= 5:\n                        R = math.sqrt(((rho_t - 4) * (rho_t - 2) * rho_inf) /\n                                      ((rho_inf - 4) * (rho_inf - 2) * rho_t))\n                        bias_correction2 = 1 - beta2_t\n                        vt.div_(bias_correction2)\n                        if nu2 != 1.0:\n                            vt.mul_(nu2).addcmul_(1 - nu2, grad, grad)\n                        denom = vt.sqrt_().add_(group['eps'])\n\n                        n = (lr * R) / denom\n\n                        if self.AdaMod:\n                            n_avg = state['n_avg']\n                            n = self.apply_AdaMod(\n                                beta3, n_avg, n, step=state['step'])\n\n                        if group['dropout'] > 0.0:\n                            mask = torch.bernoulli(\n                                torch.ones_like(p.data) - group['dropout'])\n                            n = n * mask\n\n                        p.data.add_(-n * momentum)\n                    else:\n                        if self.AdaMod:\n                            n_avg = state['n_avg']\n                            n_avg.mul_(beta3).add_(1 - beta3, lr)\n                        p.data.add_(-lr, momentum)\n                else:\n                    bias_correction2 = 1 - beta2_t\n                    vt.div_(bias_correction2)\n                    if nu2 != 1.0:\n                        vt.mul_(nu2).addcmul_(1 - nu2, grad, grad)\n                    denom = vt.sqrt_().add_(group['eps'])\n                    n = lr / denom\n                    if self.AdaMod:\n                        n_avg = state['n_avg']\n                        n = self.apply_AdaMod(\n                            beta3, n_avg, n, step=state['step'])\n\n                    if group['dropout'] > 0.0:\n                        mask = torch.bernoulli(\n                            torch.ones_like(p.data) - group['dropout'])\n                        n = n * mask\n\n                    p.data.add_(-n * momentum)\n\n                if lookahead_step:\n                    p.data.mul_(alpha).add_(\n                        1.0 - alpha, state['cached_params'])\n                    state['cached_params'].copy_(p.data)\n\n                if do_IA:\n                    p.data.add_(state[\"num_models\"], state['cached_params']\n                                ).div_(state[\"num_models\"] + 1.0)\n                    state['cached_params'].copy_(p.data)\n                    state[\"num_models\"] += 1\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"33a24846-4e5b-4dd0-89a4-17b2695bff30","_cell_guid":"04f6a6f0-4d88-4a20-9493-02baf12e4768","trusted":true},"cell_type":"code","source":"def train(train_loader, model, criterion, optimizer, epoch):\n    model.train()\n    for batch_idx, (data, target) in enumerate(train_loader):\n        # if GPU available, move data and target to GPU\n        if torch.cuda.is_available():\n            data = data.cuda()\n            target = target.cuda()\n        \n        # compute output and loss\n        output = model(data)\n        loss = criterion(output, target)\n        \n        \n        # backward and update model        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n\n        if (batch_idx + 1)% 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n                100. * (batch_idx + 1) / len(train_loader), loss.data.item()))\n        loss+=criterion(output, target).data.item()\n    loss /= len(train_loader.dataset)\n    train_losses.append(loss)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4496e2c-4007-46c0-bc67-97b94134ba4f","_cell_guid":"2da16e27-a7c2-44cc-a38f-ee3fd53af768","trusted":true},"cell_type":"code","source":"def validate(val_loader, model, criterion):\n    model.eval()\n    loss = 0\n    correct = 0\n    \n    for _, (data, target) in enumerate(val_loader):\n        if torch.cuda.is_available():\n            data = data.cuda()\n            target = target.cuda()\n        \n        output = model(data)\n        \n        loss += criterion(output, target).data.item()\n        \n        pred = output.data.max(1, keepdim=True)[1]\n        correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n        \n    loss /= len(val_loader.dataset)\n        \n    print('\\nOn Val set Average loss: {:.4f}, Accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(val_loader.dataset),\n        100.0 * float(correct) / len(val_loader.dataset)))\n    \n    valid_losses.append(loss)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"105dd984-15be-44d8-b6ae-57a1a12fa356","_cell_guid":"f97844ab-9c25-4251-be1d-8e2e7f615fdf","trusted":true},"cell_type":"code","source":"import torch.nn.functional as F\n\ndef linear_combination(x, y, epsilon): \n    return epsilon*x + (1-epsilon)*y\n\ndef reduce_loss(loss, reduction='mean'):\n    return loss.mean() if reduction=='mean' else loss.sum() if reduction=='sum' else loss\n\n\nclass LabelSmoothingCrossEntropy(nn.Module):\n    def __init__(self, epsilon:float=0.1, reduction='mean'):\n        super().__init__()\n        self.epsilon = epsilon\n        self.reduction = reduction\n    \n    def forward(self, preds, target):\n        n = preds.size()[-1]\n        log_preds = F.log_softmax(preds, dim=-1)\n        loss = reduce_loss(-log_preds.sum(dim=-1), self.reduction)\n        nll = F.nll_loss(log_preds, target, reduction=self.reduction)\n        return linear_combination(loss/n, nll, self.epsilon)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c767e258-95a1-4dd9-9124-3c7eebc81952","_cell_guid":"5371c0fe-9333-466a-839b-293b697f2948","trusted":true},"cell_type":"code","source":"# example config, use the comments to get higher accuracy\ntotal_epoches = 50 # 50\nstep_size = 5     # 10\nbase_lr = 0.01    # 0.01\nweight_decay = 0.01\nbatch_size = 64\n\n\nclass config:\n    def __init__(self):\n        self.batch_size = batch_size\n        self.wd = weight_decay\n        self.lr = base_lr\n        self.epochs = total_epoches\n        self.step_size = step_size\n\nconfig = config()\n\nstep_per_epoch = int(train_df.shape[0]/config.batch_size)\n\n#optimizer = optim.Adam(model.parameters(), lr=base_lr)\noptimizer = DemonRanger(params=model.parameters(),\n                        lr=config.lr,\n                        weight_decay=config.wd,\n                        epochs = config.epochs,\n                        step_per_epoch = step_per_epoch,\n                        betas=(0.9,0.999,0.999), # restore default AdamW betas\n                        nus=(1.0,1.0), # disables QHMomentum\n                        k=0,  # disables lookahead\n                        alpha=1.0, \n                        IA=False, # enables Iterate Averaging\n                        rectify=False, # disables RAdam Recitification\n                        AdaMod=False, #disables AdaMod\n                        AdaMod_bias_correct=False, #disables AdaMod bias corretion (not used originally)\n                        use_demon=True, #enables Decaying Momentum (DEMON)\n                        use_gc=False, #disables gradient centralization\n                        amsgrad=False # disables amsgrad\n                        )\n#criterion = nn.CrossEntropyLoss()\ncriterion = LabelSmoothingCrossEntropy()\n#criterion = nn.KLDivLoss()\n\nlr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=0.5)\n#lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,threshold=1,patience=1,mode='min')\n#lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, step_size)\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    criterion = criterion.cuda()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db770e14-bcab-4d70-8348-0bfd6d2e713a","_cell_guid":"70e87226-42c9-4a41-9f32-c2851fd40cff","trusted":true},"cell_type":"code","source":"def split_dataframe(dataframe=None, fraction=0.9, rand_seed=1):\n    df_1 = dataframe.sample(frac=fraction, random_state=rand_seed)\n    df_2 = dataframe.drop(df_1.index)\n    return df_1, df_2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f0108ec-fcf2-434c-9a6c-fb6ecbce2f90","_cell_guid":"fa5408ea-cf40-4b04-894a-e11adb0463b2","trusted":true},"cell_type":"code","source":"train_losses = []\nvalid_losses = []\n\nfor epoch in range(total_epoches):\n    print(\"\\nTrain Epoch {}: lr = {}\".format(epoch, lr_scheduler.get_last_lr()[0]))\n\n    train_df_new, val_df = split_dataframe(dataframe=train_df, fraction=0.9, rand_seed=epoch)\n    \n    \n    train_dataset = get_dataset(train_df_new, transform=train_transforms)\n    val_dataset = get_dataset(val_df, transform=val_test_transforms)\n\n    train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                               batch_size=batch_size, shuffle=True)\n    val_loader = torch.utils.data.DataLoader(dataset=val_dataset,\n                                             batch_size=batch_size, shuffle=False)\n    \n    train(train_loader=train_loader, model=model, criterion=criterion, optimizer=optimizer, epoch=epoch)\n    validate(val_loader=val_loader, model=model, criterion=criterion)\n\n    \n    lr_scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e89b218b-b5be-4b2a-82ac-c63c0978d888","_cell_guid":"68c3d46c-8277-4e5a-965b-ec9b3a0a1457","trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,8))\nplt.plot(range(1,len(train_losses)+1),train_losses, label='Training Loss')\nplt.plot(range(1,len(valid_losses)+1),valid_losses,label='Validation Loss')    \nplt.xlabel('epochs')\nplt.ylabel('loss')\nplt.xlim(0, len(train_losses)+1) # consistent scale\nplt.grid(True)\nplt.legend()\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97e52772-0860-468b-80cc-eaa05632e488","_cell_guid":"6b2c510b-9f99-4a55-ba1f-3301478c03c9","trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,8))\nplt.plot(range(1,len(train_losses)+1),train_losses, label='Training Loss')\nplt.xlabel('epochs')\nplt.ylabel('loss')\nplt.xlim(0, len(train_losses)+1) # consistent scale\nplt.grid(True)\nplt.legend()\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf4a57c5-4115-4806-8303-c12aecc62a11","_cell_guid":"3adcd758-20dd-43d9-b2fd-ff5c1d3ca237","trusted":true},"cell_type":"code","source":"def prediciton(test_loader, model):\n    model.eval()\n    test_pred = torch.LongTensor()\n    \n    for i, data in enumerate(test_loader):\n        if torch.cuda.is_available():\n            data = data.cuda()\n            \n        output = model(data)\n        \n        pred = output.cpu().data.max(1, keepdim=True)[1]\n        test_pred = torch.cat((test_pred, pred), dim=0)\n        \n    return test_pred","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb836b70-eee0-45b1-a00b-3306027f8544","_cell_guid":"5a4114c6-56c9-453d-8884-cfa696ba4219","trusted":true},"cell_type":"code","source":"test_batch_size = 64\ntest_dataset = get_dataset(test_df, transform=val_test_transforms)\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n                                          batch_size=test_batch_size, shuffle=False)\n\n# tensor prediction\ntest_pred = prediciton(test_loader, model)\n\n# tensor -> numpy.ndarray -> pandas.DataFrame\ntest_pred_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1), test_pred.numpy()], \n                      columns=['ImageId', 'Label'])\n\n# show part of prediction dataframe\nprint(test_pred_df.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efb8b88f-3eac-4225-80a9-b2e883c29de5","_cell_guid":"bc84b4d4-ff57-4a5f-bf84-6810f5021789","trusted":true},"cell_type":"code","source":"# import the modules we'll need\nfrom IPython.display import HTML\nimport base64\n\n# function that takes in a dataframe and creates a text link to  \n# download it (will only work for files < 2MB or so)\ndef create_download_link(df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv(index=False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n# create a link and click to download\ncreate_download_link(test_pred_df, filename=\"submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d577329-dbe5-47f0-b83d-ca1f0e8d83d9","_cell_guid":"a86988b2-0707-4b90-bb38-131cfbea9015","trusted":true},"cell_type":"code","source":"test_pred_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0622eb8f-f593-47a6-a65b-d97610a73a5c","_cell_guid":"2497fd30-063f-4d91-9258-9d5c57442fd2","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}