{"cells":[{"metadata":{},"cell_type":"markdown","source":"Reference : https://www.kaggle.com/hocop1/manifold-mixup-using-pytorch Ruslan Baynazarov"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split Image/Label and Normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\nvalid = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')\ntrain_img = train.iloc[:,1:].astype('float').values/255.0\ntrain_label = train.iloc[:,0].values\nvalid_img = valid.iloc[:,1:].astype('float').values/255.0\nvalid_label = valid.iloc[:,0].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_img, test_img, train_label, test_label = train_test_split(train_img, train_label, test_size=0.3, random_state=7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DataLoader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataloader(Dataset):\n    \n    def __init__(self,image,label,is_train=True):\n        \n        self.img = image\n        self.label = label\n        self.is_train = is_train\n        \n    def __len__(self):\n        return len(self.img)\n    \n    def __getitem__(self,idx):\n        \n        '''\n        Reshape: 1D) 784 -> 2D) 28x28\n        '''\n        image1 = self.img[idx].reshape(-1,28,28)\n        if self.is_train:\n            label1 = np.zeros(10, dtype='float32')\n            label1[self.label[idx]] =1\n            return image1,label1\n        else:\n            return image1\n\n\ntrainset = Dataloader(train_img,train_label)\ntestset = Dataloader(test_img,test_label)\nvalidset = Dataloader(valid_img,valid_label)\n\nbatch_size = 270\n\ntrain_loader = torch.utils.data.DataLoader(trainset,batch_size=batch_size,shuffle=True)\ntest_loader = torch.utils.data.DataLoader(testset,batch_size=batch_size,shuffle=True)\nvalid_loader = torch.utils.data.DataLoader(validset,batch_size=batch_size,shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build CNN Model"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# I know it is too heavy :)\n\nclass Block(nn.Module):\n    def __init__(self,in_channel,out_channel):\n        super(Block,self).__init__()\n        \n        '''\n        Convolution -> Max pooling -> LeakyReLU -> Dropout -> Convolution -> Max polling\n        \n        CHANNEL@HEIGHTxWIDTH\n        \n        COMPUTATION:\n        H <- (H - kernel_size + 2*padding)*1/stride + 1\n        W <- (W - kernel_size + 2*padding)*1/stride + 1\n        CH <- out_channel\n        '''\n        \n        self.block = nn.Sequential(\n            nn.Conv2d(in_channel,out_channel,kernel_size=1),\n            nn.MaxPool2d(kernel_size=3,stride=1,padding=1),\n            nn.LeakyReLU(),\n            nn.Dropout(0.4),\n            nn.Conv2d(out_channel,out_channel,kernel_size=3),\n            nn.MaxPool2d(kernel_size=3,stride=1,padding=1),\n            #nn.BatchNorm2d(out_channel)\n        )\n        \n    def forward(self,x):\n        \n        return self.block(x)\n    \nclass CNN(nn.Module):\n    def __init__(self):\n        super(CNN,self).__init__()\n        \n        self.block1 = Block(1,32)\n        self.block2 = Block(32,8)\n        #self.block3 = Block(16,8)\n        #self.batchnorm1 = nn.BatchNorm1d(512)\n        #self.batchnorm2 = nn.BatchNorm1d(32)\n        self.fc1 = nn.Linear(4608,1024)\n        self.fc2 = nn.Linear(1024,512)\n        self.fc3 = nn.Linear(512,128)\n        self.fc4 = nn.Linear(128,32)\n        self.fc5 = nn.Linear(32,10)\n\n\n    def forward(self,x):\n        \n        x = self.block1(x)\n        x = self.block2(x)\n        #x = self.block3(x)\n        x = x.view(x.size(0),-1)\n        x = F.leaky_relu(self.fc1(x))\n        x = F.leaky_relu(self.fc2(x))\n        #x = self.batchnorm1(x)\n        x = F.leaky_relu(self.fc3(x))\n        x = F.leaky_relu(self.fc4(x))\n        x = F.dropout(x)\n        #x = self.batchnorm2(x)\n        x = self.fc5(x)\n        \n        return x\nnet = CNN()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Optimizer/Loss Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\nepochs = 30\n\n'''\nAdam\n'''\noptimizer = optim.Adam(net.parameters(), lr=0.1)\n\n'''\n# Learning_rate scheduler\n'''\n#scheduler = lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)\nscheduler = lr_scheduler.ReduceLROnPlateau(optimizer,'min',patience=5,factor=0.5)\n\n'''\nGet real_time learning_rate\n'''\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return np.float(param_group['lr'])\n\n'''\nCalculate Loss (Categorical cross entropy)\n'''\ndef criterion(input, target, size_average=True):\n    \"\"\"Categorical cross-entropy with logits input and one-hot target\"\"\"\n    l = -(target * torch.log(F.softmax(input, dim=1) + 1e-10)).sum(1)\n    if size_average:\n        l = l.mean()\n    else:\n        l = l.sum()\n    return l","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EarlyStopping"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nSource : https://github.com/Bjarten/early-stopping-pytorch\n'''\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), 'checkpoint.pt')\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"net.train().double()\n'''\nTrain will be automatically stopped after 7 epochs without improvement\n'''\nearly_stopping = EarlyStopping(patience=7, verbose=True)\n\nfor epoch in range(epochs):\n    i=0\n    print(f'epoch: {int(epoch+1)}/{int(epochs)} || train/test/valid: {len(train_loader.dataset)}/{len(test_loader.dataset)}/{len(valid_loader.dataset)} ||' ' learning_rate: {:.4f}'.format(get_lr(optimizer)))\n\n    for img,label in tqdm(train_loader):\n        optimizer.zero_grad()\n        output = net(img)\n        loss = criterion(output,label.double())\n        loss.backward()\n        optimizer.step()\n        i +=1\n\n\n    net.eval()\n    with torch.no_grad():\n        correct = 0\n        accuracy = 0\n        val_correct = 0\n        val_accuracy = 0\n                \n        for data, target in test_loader:\n            output = net(data)\n            pred = output.data.max(1 , keepdim=True)[1]\n            correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).sum().numpy()\n        accuracy = correct / len(test_loader.dataset)\n                \n        for data, target in valid_loader:\n                    \n            output = net(data)\n            pred = output.data.max(1 , keepdim=True)[1]\n            val_correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).sum().numpy()\n        val_accuracy = correct / len(valid_loader.dataset)\n        print('acc: {:.2f}%||val acc: {:.2f}%'.format(accuracy*100,val_accuracy*100))\n    scheduler.step(loss)\n    early_stopping(loss, net)\n    if early_stopping.early_stop:\n        print(\"Early stopping\")\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\ntest_img = test.iloc[:,1:].astype('float').values/255.0\n\nnet.eval()\ntestset = Dataloader(test_img,None,is_train=False)\ntest_loader = torch.utils.data.DataLoader(testset,batch_size=batch_size,shuffle=False)\npredictions = []\n\nfor img in tqdm(test_loader):\n    \n    output = net(img).max(dim=1)[1] # argmax\n    predictions += list(output.data.cpu().numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}