{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Squeeze-and-Excitation Network(SE Net) for Kannada MNIST 　-　 Pytorch Implementation**\n\n* **Squeeze-and-Excitation Network: **\n\nhttps://arxiv.org/abs/1709.01507\n\nhttps://towardsdatascience.com/review-senet-squeeze-and-excitation-network-winner-of-ilsvrc-2017-image-classification-a887b98b2883\n\nhttps://medium.com/@konpat/squeeze-and-excitation-networks-hu-et-al-2017-48e691d3fe5e\n\nhttps://www.kaggle.com/c/tgs-salt-identification-challenge/discussion/65939 (A simple SE Block implementation)\n\n![](https://raw.githubusercontent.com/titu1994/keras-squeeze-excite-network/master/images/squeeze-excite-block.JPG)\n\nThis block helps dynamically “excite” feature maps that help classification and suppress feature maps that don’t help based on the patterns of global \naverages of feature maps\n\n* **Squeeze-and-Excitation (SE) Block**\n![](https://miro.medium.com/max/353/1*C7vDgQ2ce3k1_gO7345WuA.png)"},{"metadata":{},"cell_type":"markdown","source":"**Import Packages**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torchvision import transforms, datasets\nfrom torch.utils.data import DataLoader, Dataset\nfrom PIL import Image\nimport matplotlib.pyplot as plt\ndevice = \"cuda\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define the Model**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\nclass Sq_Ex_Block(nn.Module):\n    def __init__(self, in_ch, r):\n        super(Sq_Ex_Block, self).__init__()\n        self.se = nn.Sequential(\n            GlobalAvgPool(),\n            nn.Linear(in_ch, in_ch//r),\n            nn.ReLU(inplace=True),\n            nn.Linear(in_ch//r, in_ch),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        se_weight = self.se(x).unsqueeze(-1).unsqueeze(-1)\n        return x.mul(se_weight)\n\nclass GlobalAvgPool(nn.Module):\n    def __init__(self):\n        super(GlobalAvgPool, self).__init__()\n    def forward(self, x):\n        return x.view(*(x.shape[:-2]),-1).mean(-1)\n\nclass SE_Net(nn.Module):\n    def __init__(self,in_channels):\n        super(SE_Net,self).__init__()\n        #torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, \n        #                dilation=1, groups=1, bias=True, padding_mode='zeros')\n        self.c1 = nn.Conv2d(in_channels=in_channels, out_channels=64,kernel_size=3,stride=1,padding=0)\n        self.bn1 = nn.BatchNorm2d(num_features=64,eps=1e-3,momentum=0.01)\n        self.c2 = nn.Conv2d(64,64,3,1,0)\n        self.bn2 = nn.BatchNorm2d(64,1e-3,0.01)\n        self.c3 = nn.Conv2d(64,64,5,1,2)\n        self.bn3 = nn.BatchNorm2d(64,1e-3,0.01)\n        self.m1 = nn.MaxPool2d(2)\n        self.d1 = nn.Dropout(0.4)\n        \n        self.c4 = nn.Conv2d(64,128,3,1,0)\n        self.bn4 = nn.BatchNorm2d(128,1e-3,0.01)\n        self.c5 = nn.Conv2d(128,128,3,1,0)\n        self.bn5 = nn.BatchNorm2d(128,1e-3,0.01)\n        self.c6 = nn.Conv2d(128,128,5,1,2)\n        self.bn6 = nn.BatchNorm2d(128,1e-3,0.01)\n        self.m2 = nn.MaxPool2d(2)\n        self.d2 = nn.Dropout(0.4)\n        \n        self.c7 = nn.Conv2d(128,256,3,1,0)\n        self.bn7 = nn.BatchNorm2d(256,1e-3,0.01)\n        self.se3 = Sq_Ex_Block(in_ch=256,r=8)\n        self.m3 = nn.MaxPool2d(2)\n        self.d3 = nn.Dropout(0.4)\n\n        self.fc1 = nn.Linear(256*1*1,256)\n        self.bn8 = nn.BatchNorm1d(256,1e-3,0.01)\n        \n        self.out = nn.Linear(256,10)\n        \n        self.init_linear_weights()\n        \n    def forward(self,x):\n        x = self.bn1(F.leaky_relu(self.c1(x),0.1))\n        x = self.bn2(F.leaky_relu(self.c2(x),0.1))\n        x = self.bn3(F.leaky_relu(self.c3(x),0.1))\n        x = self.d1(self.m1(x))\n        \n        x = self.bn4(F.leaky_relu(self.c4(x),0.1))\n        x = self.bn5(F.leaky_relu(self.c5(x),0.1))\n        x = self.bn6(F.leaky_relu(self.c6(x),0.1))\n        x = self.d2(self.m2(x))\n        \n        x = self.bn7(F.leaky_relu(self.c7(x),0.1))\n        x = self.se3(x)\n        x = self.d3(self.m3(x))\n        \n        x = x.view(-1, 256*1*1) #reshape\n        x = self.bn8(F.leaky_relu(self.fc1(x),0.1))\n        return self.out(x)\n    \n    def init_linear_weights(self):\n        nn.init.kaiming_normal_(self.fc1.weight, mode='fan_in')  #default mode: fan_in\n        nn.init.kaiming_normal_(self.out.weight, mode='fan_in')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define the Transformation Function for Data Augmentation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"trans = transforms.Compose([\n        transforms.RandomAffine(degrees=10,translate=(0.15,0.15),scale=[0.9,1.1],shear=5), #Data augmentation\n        transforms.ToTensor(),  #Take Image as input and convert to tensor with value from 0 to1  \n    ])\ntrans_val = transforms.Compose([\n        transforms.ToTensor(),  #Take Image as input and convert to tensor with value from 0 to1\n    ])\ntrans_test = transforms.Compose([\n        transforms.ToTensor(),  #Take Image as input and convert to tensor with value from 0 to1\n    ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define the Dataset**"},{"metadata":{"trusted":true},"cell_type":"code","source":"global_data = pd.read_csv(\"/kaggle/input/Kannada-MNIST/train.csv\")\nglobal_data_test = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\")\n\nclass KMnistDataset(Dataset):\n    def __init__(self,data_len=None, is_validate=False,validate_rate=None,indices=None, data=None):\n        self.is_validate = is_validate\n        self.data = global_data\n        if data_len == None:\n            data_len = len(self.data)\n        \n        self.indices = indices\n        if self.is_validate:\n            self.len = int(data_len*validate_rate)\n            self.offset = int(data_len*(1-validate_rate))\n            self.transform = trans_val\n        else:\n            self.len = int(data_len*(1-validate_rate))\n            self.offset = 0\n            self.transform = trans\n        \n    def __getitem__(self, idx):\n        idx += self.offset\n        idx = self.indices[idx]\n        img = self.data.iloc[idx, 1:].values.astype(np.uint8).reshape((28, 28))  #value: 0~255\n        label = self.data.iloc[idx, 0]  #shape: (num,)\n        img = Image.fromarray(img)\n        img = self.transform(img)     #value: 0~1, shape:(1,28,28)\n        label = torch.as_tensor(label, dtype=torch.uint8)    #value: 0~9, shape(1)\n        return img, label\n\n    def __len__(self):\n        return self.len\n    \nclass TestDataset(Dataset):\n    def __init__(self,data_len=None):\n        self.data = global_data_test\n        self.transform = trans_test\n        if data_len == None:\n            self.len = len(self.data)\n        \n    def __getitem__(self, idx):\n        img = self.data.iloc[idx, 1:].values.astype(np.uint8).reshape((28, 28))  #value: 0~255\n        img = Image.fromarray(img)\n        img = self.transform(img)     #value: 0~1, shape:(1,28,28)\n        return img, torch.Tensor([])\n\n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Hyperparameter**"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 1024\nnum_workers = 8\nepochs = 70\nlr = 1e-3\nval_period = 1\nval_rate = 0.1    ###Train->54000 images, Validation->6000 images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define the Criterion, Optimizer and lr_scheduler**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = SE_Net(in_channels=1)\nif device == \"cuda\":\n    model.cuda()\n    \ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(),lr=lr,betas=(0.9,0.99))\nlr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, verbose=True, patience=15,factor=0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I use Adam as optimizer and ReduceLROnPlateau to adjust the learning rate (according to the validation errors)\n\nThe learning rate curve will be like:\n![](https://3qeqpr26caki16dnhd19sv6by6v-wpengine.netdna-ssl.com/wp-content/uploads/2018/11/Line-Plots-of-Learning-Rate-Over-Epochs-for-Different-Patience-Values-Used-in-the-ReduceLROnPlateau-Schedule.png)\n\nLink:\nhttps://machinelearningmastery.com/understand-the-dynamics-of-learning-rate-on-deep-learning-neural-networks/"},{"metadata":{},"cell_type":"markdown","source":"**Prepare the Dataloader**"},{"metadata":{"trusted":true},"cell_type":"code","source":"indices_len = len(global_data)  ###For dataset\nindices = np.arange(indices_len)\n\ntrain_dataset = KMnistDataset(data_len=None,is_validate=False, validate_rate=val_rate,indices=indices)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n\nval_dataset = KMnistDataset(data_len=None,is_validate=True, validate_rate=val_rate, indices=indices)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Train the Model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"min_loss = 10000\nmax_acc = 0\nbest_model_dict = None\n\nprint(\"Start training...\")\nfor ep in range(0,epochs+1):\n    model.train()\n    data_num = 0\n    \n    ###Train\n    for idx, data in enumerate(train_loader):\n        img, target = data\n        img, target = img.to(device), target.to(device,dtype=torch.long)\n\n        pred = model(img)\n        loss = criterion(pred,target)\n        data_num += img.size(0)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n    ###Validate\n    if ep!=0 and ep%val_period == 0:\n        model.eval()\n        acc = 0\n        val_loss = 0\n        data_num  = 0\n        with torch.no_grad():\n            for idx, data in enumerate(val_loader):\n                img, target = data\n                img, target = img.to(device), target.to(device,dtype=torch.long)\n                \n                pred = model(img)\n                val_loss += criterion(pred, target).item()\n                _,pred_class = torch.max(pred.data, 1)\n                acc += (pred_class == target).sum().item()\n                data_num += img.size(0)\n        \n        acc /= data_num\n        val_loss /= data_num\n\n        ###Reduce learning rate and Early stopping\n        lr_scheduler.step(val_loss)\n        if optimizer.param_groups[0]['lr'] < 1e-4:\n            break                    \n\n        ###Save the best model\n        if acc >= max_acc:\n            max_acc = acc\n            min_loss = val_loss\n            best_model_dict = model.state_dict()                    \n\n        print(\"Episode:{}, Validation Loss:{},Accuracy:{:.4f}%,learning rate:{}\"\n              .format(ep,val_loss,acc*100,optimizer.param_groups[0]['lr']))\n    \nprint(\"===================Best Model, Loss:{} Accuracy:{}==================\".format(min_loss,max_acc))\nprint(\"====================================================================\")\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Inference the Test Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"result = np.array([],dtype=np.int)\ntest_dataset = TestDataset(data_len=None)\ntest_loader = DataLoader(test_dataset, batch_size=256, shuffle=False, num_workers=8)\n\ntest_model = SE_Net(in_channels=1)\ntest_model.load_state_dict(best_model_dict)\nif device == \"cuda\":\n    test_model.cuda()\ntest_model.eval()\n\nwith torch.no_grad():\n    for idx, data in enumerate(test_loader):\n        img = data[0].to(device)\n        pred = test_model(img)\n        _,pred_class = torch.max(pred.data, 1)\n        result = np.concatenate([result,pred_class.cpu().numpy()],axis=0)\nprint(\"shape of the result:\",np.shape(result))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Submit the Answer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub=pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv')\nsample_sub['label']=result\nsample_sub.to_csv('submission.csv',index=False)\nsample_sub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n** I've learned a lot from others discussions and kernels, and I hope this information will be helpful.**\n\n** Thanks for reading! **"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}