{"cells":[{"metadata":{},"cell_type":"markdown","source":"# PyTorch Kanada MNIST \n_2019-10-29 Mengcius_\n\n**Additions:** <br>\n1. Model: ResNet, EfficientNet\n2. Optimizer: Adam, RAdam, AdamW, Adadelta\n3. Loss: CrossEntropyLoss, FocalLoss, CrossEntropyLabelSmooth\n4. [LR Scheduler](https://github.com/mengcius/PyTorch-Learning-Rate-Scheduler): StepLR, CosineAnnealingLR, ExponentialLR, ReduceLROnPlateau, CosineAnnealingWarmRestarts\n\n[GitHub](http://github.com/mengcius) \n\n[知乎](https://www.zhihu.com/people/mengcius/columns)\n\nfork https://www.kaggle.com/nicapotato/pytorch-cnn-kanada"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !wget https://mengcius.coding.net/api/share/download/ad2a6c94-1036-409a-a4bf-9cef4088e990 -O ./Kannada-MNIST.zip\n# !unzip Kannada-MNIST.zip","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# Any results you write to the current directory are saved as output.\n\nimport torch\nimport torchvision\nfrom torchvision import transforms, datasets\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\nimport math\nimport random\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\nprint(torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset process "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data\ntrain=pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest=pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')\nsubmission_set = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\").iloc[:,1:]\n\ntrain_data=train.drop('label',axis=1)\ntrain_targets=train['label']\n\ntest_images=test.drop('label',axis=1)\ntest_labels=test['label']\n\n# Train Test Split\ntrain_images, val_images, train_labels, val_labels = train_test_split(train_data, \n                                                                     train_targets, \n                                                                     test_size=0.2)\n\n# Reset Index\ntrain_images.reset_index(drop=True, inplace=True)\ntrain_labels.reset_index(drop=True, inplace=True)\n\nval_images.reset_index(drop=True, inplace=True)\nval_labels.reset_index(drop=True, inplace=True)\n\ntest_images.reset_index(drop=True, inplace=True)\ntest_labels.reset_index(drop=True, inplace=True)\n\nprint(\"Train Set\")\nprint(train_images.shape)\nprint(train_labels.shape)\n\nprint(\"Validation Set\")\nprint(val_images.shape)\nprint(val_labels.shape)\n\nprint(\"Validation 2\")\nprint(test_images.shape)\nprint(test_labels.shape)\n\nprint(\"Submission\")\nprint(submission_set.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Look at image means\")\nprint(train_images.mean(axis = 1).mean())\nprint(val_images.mean(axis = 1).mean())\nprint(test_images.mean(axis = 1).mean())\nprint(submission_set.mean(axis = 1).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Distribution\")\nprint(train_labels.value_counts(normalize = True))\n\nprint(\"\\nSubmission Distribution\")\nprint(test_labels.value_counts(normalize = True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMGSIZE = 28\n\n# Transformations for the train\ntrain_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.RandomCrop(IMGSIZE),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1)), # 保持图像中心不变的随机仿射变换\n#     transforms.RandomRotation(10), # 随机旋转(-degrees， +degrees)\n#     transforms.RandomErasing(p=0.5, scale=(0.01, 0.02), ratio=(0.3, 3.3)), # 随机擦除\n#     transforms.ColorJitter(), # 随机改变图像的亮度、对比度和饱和度\n    transforms.ToTensor(), # divides by 255\n  #  transforms.Normalize((0.5,), (0.5,))\n]))\n\n# Transformations for the validation & test sets\nval_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.ToTensor(), # divides by 255\n   # transforms.Normalize((0.1307,), (0.3081,))\n]))\n\nclass KannadaDataSet(torch.utils.data.Dataset):\n    def __init__(self, images, labels,transforms = None):\n        self.X = images\n        self.y = labels\n        self.transforms = transforms\n         \n    def __len__(self):\n        return (len(self.X))\n    \n    def __getitem__(self, i):\n        data = self.X.iloc[i,:]\n        data = np.array(data).astype(np.uint8).reshape(IMGSIZE,IMGSIZE,1)\n        \n        if self.transforms:\n            data = self.transforms(data)\n            \n        if self.y is not None:\n            return (data, self.y[i])\n        else:\n            return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\ntrain_data = KannadaDataSet(train_images, train_labels, train_trans)\nval_data = KannadaDataSet(val_images, val_labels, val_trans)\ntest_data = KannadaDataSet(test_images, test_labels, val_trans)\nsubmission_data = KannadaDataSet(submission_set, None, val_trans)\n\n\ntrain_loader = torch.utils.data.DataLoader(train_data, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\nval_loader = torch.utils.data.DataLoader(val_data, \n                                           batch_size=batch_size, \n                                           shuffle=False)\n\ntest_loader = torch.utils.data.DataLoader(test_data,  # Dig-MNIST.csv\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_data,\n                                          batch_size=batch_size, \n                                          shuffle=False)\n\nclasses = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Build CNN model"},{"metadata":{},"cell_type":"markdown","source":"nicapotato DenseNet "},{"metadata":{"trusted":true},"cell_type":"code","source":"class DenseNet(nn.Module):\n    def __init__(self, dropout = 0.40):\n        super(DenseNet, self).__init__()\n        self.dropout = dropout\n        \n        # https://blog.algorithmia.com/convolutional-neural-nets-in-pytorch\n        #Our batch shape for input x is (1, 28, 28)\n        # (Batch, Number Channels, height, width).\n        #Input channels = 1, output channels = 18\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=5, stride=1, padding=2)\n        self.conv1_bn = nn.BatchNorm2d(num_features=64)\n        \n        self.conv1_1 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=1, padding=2)\n        self.conv1_1_bn = nn.BatchNorm2d(num_features=64)\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_1 = nn.Dropout2d(p=self.dropout)\n        \n        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)\n        self.conv2_bn = nn.BatchNorm2d(num_features=128)\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_2 = nn.Dropout2d(p=self.dropout)\n        \n        self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)\n        self.conv3_bn = nn.BatchNorm2d(num_features=256)\n        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.d2_3 = nn.Dropout2d(p=self.dropout)\n        \n        #4608 input features, 256 output features (see sizing flow below)\n        self.fc1 = nn.Linear(256 * 3 * 3, 512) # Linear 1\n        self.d1_1 = nn.Dropout(p=self.dropout)\n        #64 input features, 10 output features for our 10 defined classes\n        self.fc2 = nn.Linear(in_features=512, out_features=256) # linear 2\n        self.d1_2 = nn.Dropout(p=self.dropout)\n        self.fc3 = nn.Linear(in_features=256, out_features=128) # linear 3\n        self.d1_3 = nn.Dropout(p=self.dropout)\n        self.out = nn.Linear(in_features=128, out_features=10) # linear 3\n        \n    def forward(self, x):\n        #Computes the activation of the first convolution\n        #Size changes from (1, 28, 28) to (18, 28, 28)\n        x = self.conv1(x)\n        x = self.conv1_bn(x)\n        x = F.relu(x)\n        x = self.conv1_1(x)\n        x = self.conv1_1_bn(x)\n        x = F.relu(x)       \n        \n        x = self.d2_1(x)\n        x = self.pool1(x) # Size changes from (18, 28, 28) to (18, 14, 14)\n        \n        # Second Conv       \n        x = self.conv2(x)\n        x = self.conv2_bn(x)\n        x = F.relu(x)\n        x = self.d2_2(x)\n        x = self.pool2(x) # Size changes from (18, 14, 14) to (18, 7, 7)\n        \n        # Third Conv       \n        x = self.conv3(x)\n        x = self.conv3_bn(x)\n        x = F.relu(x)\n        x = self.d2_3(x)\n        x = self.pool3(x) # Size changes from (18, 7, 7) to (18, 3, 3)\n        \n        #Reshape data to input to the input layer of the neural net\n        #Size changes from (18, 14, 14) to (1, 3528)\n        #Recall that the -1 infers this dimension from the other given dimension\n        x = x.view(-1, 256 * 3 * 3)\n\n        #Computes the activation of the first fully connected layer\n        #Size changes from (1, 4608) to (1, 64)\n        #Computes the second fully connected layer (activation applied later)\n        #Size changes from (1, 64) to (1, 10)\n        x = F.relu(self.fc1(x))\n        x = self.d1_1(x)\n        \n        x = F.relu(self.fc2(x))\n        x = self.d1_2(x)\n        \n        x = F.relu(self.fc3(x))\n        x = self.d1_3(x)\n        \n        x = self.out(x)\n        return F.log_softmax(x, dim=-1)\n\n# net = DenseNet().to(device)\n# net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def outputSize(in_size, kernel_size, stride, padding):\n    output = int((in_size - kernel_size + 2 * (padding)) / stride) + 1\n    return(output)\n# outputSize(64, 5, 1, 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ResNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''ResNet in PyTorch.\n\nFor Pre-activation ResNet, see 'preact_resnet.py'.\n\nReference:\n[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun\n    Deep Residual Learning for Image Recognition. arXiv:1512.03385\n'''\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, in_planes, planes, stride=1):\n        super(BasicBlock, self).__init__()\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != self.expansion*planes:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(self.expansion*planes)\n            )\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        out += self.shortcut(x)\n        out = F.relu(out)\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, in_planes, planes, stride=1):\n        super(Bottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(self.expansion*planes)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != self.expansion*planes:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(self.expansion*planes)\n            )\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = F.relu(self.bn2(self.conv2(out)))\n        out = self.bn3(self.conv3(out))\n        out += self.shortcut(x)\n        out = F.relu(out)\n        return out\n\n\nclass ResNet(nn.Module):\n    def __init__(self, block, num_blocks, num_classes=10):\n        super(ResNet, self).__init__()\n        self.in_planes = 64\n\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n        self.linear = nn.Linear(512*block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, num_blocks, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, stride))\n            self.in_planes = planes * block.expansion\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.layer4(out)\n        out = F.avg_pool2d(out, 4)\n        out = out.view(out.size(0), -1)\n        out = self.linear(out)\n        return out\n\n\ndef res_net18():\n    return ResNet(BasicBlock, [2,2,2,2])\n\ndef res_net34():\n    return ResNet(BasicBlock, [3,4,6,3])\n\ndef res_net50():\n    return ResNet(Bottleneck, [3,4,6,3])\n\ndef res_net101():\n    return ResNet(Bottleneck, [3,4,23,3])\n\ndef res_net152():\n    return ResNet(Bottleneck, [3,8,36,3])\n\n\ndef test():\n    net = res_net18()#.to(device)\n    y = net(torch.randn(1,1,28,28))\n    print(y.size())\n\ntest()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Efficientnet"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch import nn\n\ntry:\n    from torch.hub import load_state_dict_from_url\nexcept ImportError:\n    from torch.utils.model_zoo import load_url as load_state_dict_from_url\n\nmodel_urls = {\n    'efficientnet_b0': 'https://www.dropbox.com/s/9wigibun8n260qm/efficientnet-b0-4cfa50.pth?dl=1',\n    'efficientnet_b1': 'https://www.dropbox.com/s/6745ear79b1ltkh/efficientnet-b1-ef6aa7.pth?dl=1',\n    'efficientnet_b2': 'https://www.dropbox.com/s/0dhtv1t5wkjg0iy/efficientnet-b2-7c98aa.pth?dl=1',\n    'efficientnet_b3': 'https://www.dropbox.com/s/5uqok5gd33fom5p/efficientnet-b3-bdc7f4.pth?dl=1',\n    'efficientnet_b4': 'https://www.dropbox.com/s/y2nqt750lixs8kc/efficientnet-b4-3e4967.pth?dl=1',\n    'efficientnet_b5': 'https://www.dropbox.com/s/qxonlu3q02v9i47/efficientnet-b5-4c7978.pth?dl=1',\n    'efficientnet_b6': None,\n    'efficientnet_b7': None,\n}\n\nparams = {\n    'efficientnet_b0': (1.0, 1.0, 224, 0.2),\n    'efficientnet_b1': (1.0, 1.1, 240, 0.2),\n    'efficientnet_b2': (1.1, 1.2, 260, 0.3),\n    'efficientnet_b3': (1.2, 1.4, 300, 0.3),\n    'efficientnet_b4': (1.4, 1.8, 380, 0.4),\n    'efficientnet_b5': (1.6, 2.2, 456, 0.4),\n    'efficientnet_b6': (1.8, 2.6, 528, 0.5),\n    'efficientnet_b7': (2.0, 3.1, 600, 0.5),\n}\n\n\nclass Swish(nn.Module):\n\n    def __init__(self, *args, **kwargs):\n        super(Swish, self).__init__()\n\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\nclass ConvBNReLU(nn.Sequential):\n\n    def __init__(self, in_planes, out_planes, kernel_size, stride=1, groups=1):\n        padding = self._get_padding(kernel_size, stride)\n        super(ConvBNReLU, self).__init__(\n            nn.ZeroPad2d(padding),\n            nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding=0, groups=groups, bias=False),\n            nn.BatchNorm2d(out_planes),\n            Swish(),\n        )\n\n    def _get_padding(self, kernel_size, stride):\n        p = max(kernel_size - stride, 0)\n        return [p // 2, p - p // 2, p // 2, p - p // 2]\n\n\nclass SqueezeExcitation(nn.Module):\n\n    def __init__(self, in_planes, reduced_dim):\n        super(SqueezeExcitation, self).__init__()\n        self.se = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Conv2d(in_planes, reduced_dim, 1),\n            Swish(),\n            nn.Conv2d(reduced_dim, in_planes, 1),\n            nn.Sigmoid(),\n        )\n\n    def forward(self, x):\n        return x * self.se(x)\n\n\nclass MBConvBlock(nn.Module):\n\n    def __init__(self,\n                 in_planes,\n                 out_planes,\n                 expand_ratio,\n                 kernel_size,\n                 stride,\n                 reduction_ratio=4,\n                 drop_connect_rate=0.2):\n        super(MBConvBlock, self).__init__()\n        self.drop_connect_rate = drop_connect_rate\n        self.use_residual = in_planes == out_planes and stride == 1\n        assert stride in [1, 2]\n        assert kernel_size in [3, 5]\n\n        hidden_dim = in_planes * expand_ratio\n        reduced_dim = max(1, int(in_planes / reduction_ratio))\n\n        layers = []\n        # pw\n        if in_planes != hidden_dim:\n            layers += [ConvBNReLU(in_planes, hidden_dim, 1)]\n\n        layers += [\n            # dw\n            ConvBNReLU(hidden_dim, hidden_dim, kernel_size, stride=stride, groups=hidden_dim),\n            # se\n            SqueezeExcitation(hidden_dim, reduced_dim),\n            # pw-linear\n            nn.Conv2d(hidden_dim, out_planes, 1, bias=False),\n            nn.BatchNorm2d(out_planes),\n        ]\n\n        self.conv = nn.Sequential(*layers)\n\n    def _drop_connect(self, x):\n        if not self.training:\n            return x\n        keep_prob = 1.0 - self.drop_connect_rate\n        batch_size = x.size(0)\n        random_tensor = keep_prob\n        random_tensor += torch.rand(batch_size, 1, 1, 1, device=x.device)\n        binary_tensor = random_tensor.floor()\n        return x.div(keep_prob) * binary_tensor\n\n    def forward(self, x):\n        if self.use_residual:\n            return x + self._drop_connect(self.conv(x))\n        else:\n            return self.conv(x)\n\n\ndef _make_divisible(value, divisor=8):\n    new_value = max(divisor, int(value + divisor / 2) // divisor * divisor)\n    if new_value < 0.9 * value:\n        new_value += divisor\n    return new_value\n\n\ndef _round_filters(filters, width_mult):\n    if width_mult == 1.0:\n        return filters\n    return int(_make_divisible(filters * width_mult))\n\n\ndef _round_repeats(repeats, depth_mult):\n    if depth_mult == 1.0:\n        return repeats\n    return int(math.ceil(depth_mult * repeats))\n\n\nclass EfficientNet(nn.Module):\n\n    def __init__(self, width_mult=1.0, depth_mult=1.0, dropout_rate=0.2, num_classes=10):\n        super(EfficientNet, self).__init__()\n\n        # yapf: disable\n        settings = [\n            # t,  c, n, s, k\n            [1,  16, 1, 1, 3],  # MBConv1_3x3, SE, 112 -> 112\n            [6,  24, 2, 2, 3],  # MBConv6_3x3, SE, 112 ->  56\n            [6,  40, 2, 2, 5],  # MBConv6_5x5, SE,  56 ->  28\n            [6,  80, 3, 2, 3],  # MBConv6_3x3, SE,  28 ->  14\n            [6, 112, 3, 1, 5],  # MBConv6_5x5, SE,  14 ->  14\n            [6, 192, 4, 2, 5],  # MBConv6_5x5, SE,  14 ->   7\n            [6, 320, 1, 1, 3]   # MBConv6_3x3, SE,   7 ->   7\n        ]\n        # yapf: enable\n\n        out_channels = _round_filters(32, width_mult)\n        features = [ConvBNReLU(1, out_channels, 3, stride=1)] # (3, out_channels, 3, stride=2)\n\n        in_channels = out_channels\n        for t, c, n, s, k in settings:\n            out_channels = _round_filters(c, width_mult)\n            repeats = _round_repeats(n, depth_mult)\n            for i in range(repeats):\n                stride = s if i == 0 else 1\n                features += [MBConvBlock(in_channels, out_channels, expand_ratio=t, stride=stride, kernel_size=k)]\n                in_channels = out_channels\n\n        last_channels = _round_filters(1280, width_mult)\n        features += [ConvBNReLU(in_channels, last_channels, 1)]\n\n        self.features = nn.Sequential(*features)\n        self.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(last_channels, num_classes),\n        )\n\n        # weight initialization\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out')\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.Linear):\n                fan_out = m.weight.size(0)\n                init_range = 1.0 / math.sqrt(fan_out)\n                nn.init.uniform_(m.weight, -init_range, init_range)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.mean([2, 3])\n        x = self.classifier(x)\n        return x\n\n\ndef _efficientnet(arch, pretrained, progress, **kwargs):\n    width_mult, depth_mult, _, dropout_rate = params[arch]\n    model = EfficientNet(width_mult, depth_mult, dropout_rate, **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch], progress=progress)\n\n        if 'num_classes' in kwargs and kwargs['num_classes'] != 1000:\n            del state_dict['classifier.1.weight']\n            del state_dict['classifier.1.bias']\n\n        model.load_state_dict(state_dict, strict=False)\n    return model\n\n\ndef efficientnet_b0(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b0', pretrained, progress, **kwargs)\n\ndef efficientnet_b1(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b1', pretrained, progress, **kwargs)\n\ndef efficientnet_b2(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b2', pretrained, progress, **kwargs)\n\ndef efficientnet_b3(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b3', pretrained, progress, **kwargs)\n\ndef efficientnet_b4(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b4', pretrained, progress, **kwargs)\n\ndef efficientnet_b5(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b5', pretrained, progress, **kwargs)\n\ndef efficientnet_b6(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b6', pretrained, progress, **kwargs)\n\ndef efficientnet_b7(pretrained=False, progress=True, **kwargs):\n    return _efficientnet('efficientnet_b7', pretrained, progress, **kwargs)\n\n\ndef test1():\n    net = efficientnet_b0()#.to(device)\n    y = net(torch.randn(2,1,28,28))\n    print(y.size())\n\n\n# test1()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RAdam"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:                    \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss\n    \niter_idx = 0    \nclass AdamW(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\n                 weight_decay=0, use_variance=True, warmup = 4000):\n        defaults = dict(lr=lr, betas=betas, eps=eps,\n                        weight_decay=weight_decay, use_variance=True, warmup = warmup)\n        print('======== Warmup: {} ========='.format(warmup))\n        super(AdamW, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(AdamW, self).__setstate__(state)\n\n    def step(self, closure=None):\n        global iter_idx\n        iter_idx += 1\n        grad_list = list()\n        mom_list = list()\n        mom_2rd_list = list()\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                state['step'] += 1\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                denom = exp_avg_sq.sqrt().add_(group['eps'])\n                bias_correction1 = 1 - beta1 ** state['step']\n                bias_correction2 = 1 - beta2 ** state['step']\n                \n                if group['warmup'] > state['step']:\n                    scheduled_lr = 1e-6 + state['step'] * (group['lr'] - 1e-6) / group['warmup']\n                else:\n                    scheduled_lr = group['lr']\n\n                step_size = scheduled_lr * math.sqrt(bias_correction2) / bias_correction1\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32)\n\n                p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### FocalLoss"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.autograd import Variable\n\nclass FocalLoss(nn.Module):\n    def __init__(self,reduction='mean',alpha=0.01,gamma=1):\n        super(FocalLoss,self).__init__()\n        self.reduction = reduction\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self,ypred,ytrue):\n        logpt = F.log_softmax(ypred,1) \n        pt = Variable(torch.exp(logpt))\n        ytrue = to_categorical(ytrue)\n        pt_prime = 1 - pt\n        focalloss = -self.alpha * (pt_prime)**self.gamma * ytrue * logpt\n        focalloss = torch.sum(focalloss,1)\n        if(self.reduction=='sum'):\n            return focalloss.sum()\n        else:\n            return focalloss.mean()\n\nnumclasses = 10\n\ndef to_categorical(ytrue):\n    input_shape = ytrue.size()\n    n = ytrue.size(0)\n    categorical = torch.zeros(n, numclasses).to(device)\n    categorical[torch.arange(n), ytrue] = 1\n    output_shape = input_shape + (numclasses,)\n    categorical = torch.reshape(categorical, output_shape)\n    return categorical","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CrossEntropyLabelSmooth"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CrossEntropyLabelSmooth(nn.Module):\n    \n    def __init__(self, num_classes, epsilon=0.1, use_gpu=True):\n        super(CrossEntropyLabelSmooth, self).__init__()\n        self.num_classes = num_classes\n        self.epsilon = epsilon\n        self.use_gpu = use_gpu\n        self.logsoftmax = nn.LogSoftmax(dim=1)\n\n    def forward(self, inputs, targets):\n        log_probs = self.logsoftmax(inputs)\n        targets = torch.zeros(log_probs.size()).scatter_(1, targets.unsqueeze(1).data.cpu(), 1)\n        if self.use_gpu: targets = targets.cuda()\n        targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes\n        loss = (- targets * log_probs).mean(0).sum()\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Find init lr"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learning Rate Finder https://sgugger.github.io/how-do-you-find-a-good-learning-rate.html\ndef find_lr(trn_loader, init_value = 1e-8, final_value=10., beta = 0.98):\n    num = len(trn_loader)-1\n    mult = (final_value / init_value) ** (1/num)\n    lr = init_value\n    optimizer.param_groups[0]['lr'] = lr\n    avg_loss = 0.\n    best_loss = 0.\n    batch_num = 0\n    losses = []\n    log_lrs = []\n    for data in trn_loader:\n        batch_num += 1\n        #As before, get the loss for this mini-batch of inputs/outputs\n        inputs = data[0].to(device)\n        labels = data[1].to(device)\n        optimizer.zero_grad()\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        #Compute the smoothed loss\n        avg_loss = beta * avg_loss + (1-beta)*loss.item()\n        smoothed_loss = avg_loss / (1 - beta**batch_num)\n        #Stop if the loss is exploding\n        if batch_num > 1 and smoothed_loss > 4 * best_loss:\n            return log_lrs, losses\n        #Record the best loss\n        if smoothed_loss < best_loss or batch_num==1:\n            best_loss = smoothed_loss\n        #Store the values\n        losses.append(smoothed_loss)\n        log_lrs.append(math.log10(lr))\n        #Do the SGD step\n        loss.backward()\n        optimizer.step()\n        #Update the lr for the next step\n        lr *= mult\n        optimizer.param_groups[0]['lr'] = lr\n    return log_lrs, losses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# net = DenseNet().to(device)\nnet = res_net18().to(device)\n# net = efficientnet_b0().to(device)\n\n# Loss Function\ncriterion = nn.CrossEntropyLoss()\n# criterion = F.nll_loss\n# criterion = FocalLoss(alpha=1,gamma=0)\n# criterion = CrossEntropyLabelSmooth(num_classes=10)\n\n# Gradient Descent\n# optimizer = optim.SGD(net.parameters(),lr=1e-1)\noptimizer = optim.Adam(net.parameters(), lr=1e-4)\n# optimizer = RAdam(net.parameters(), lr=1e-1, betas=(0.9, 0.999))\n# optimizer = AdamW(net.parameters(), lr=1e-3, betas=(0.9, 0.999), warmup = 4000)\n# optimizer = optim.Adadelta(net.parameters(), lr=0.1)\n\n%matplotlib inline\nlogs,losses = find_lr(trn_loader = train_loader)\nplt.plot(logs[10:-5],losses[10:-5])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.manual_seed(1234) \ntorch.cuda.manual_seed(1234)\ntorch.cuda.manual_seed_all(1234)  # multi-GPU\nnp.random.seed(1234)  # Numpy module\nrandom.seed(1234)  # Python random module\ntorch.backends.cudnn.benchmark = False\ntorch.backends.cudnn.deterministic = True\n\nEPOCHS = 80\nnn_output = []\n\n# net = DenseNet().to(device)\nnet = res_net18().to(device)\n# net = efficientnet_b0().to(device)\n\n# optimizer = optim.SGD(net.parameters(),lr=1e-2)\noptimizer = optim.Adam(net.parameters(), lr=1e-4)\n# optimizer = RAdam(net.parameters(), lr=1e-3, betas=(0.9, 0.999))\n# optimizer = AdamW(net.parameters(), lr=1e-3, betas=(0.9, 0.999), warmup = 4000)\n# optimizer = optim.Adadelta(net.parameters(), lr=0.1)\n\n# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=EPOCHS//4, gamma=0.1)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min = 0)\n# scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=5) # ReduceLROnPlateau里没有scheduler.get_lr()\n# scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=2, eta_min = 0) # torch===1.3.0\n\ncriterion = nn.CrossEntropyLoss()\n# criterion = F.nll_loss\n# criterion = FocalLoss(alpha=1,gamma=0)\n# criterion = CrossEntropyLabelSmooth(num_classes=10)\n\n\ndef get_num_correct(preds, labels):\n    return preds.argmax(dim=1).eq(labels).sum().item()\n\nfor epoch in range(EPOCHS):\n    epoch_loss = 0\n    epoch_correct = 0\n    net.train()\n    \n    for data in train_loader:\n        # `data` is a batch of data\n        # Before using transforms, I used .unsqueeze(1) to enter a empty number channel array (Batch, Number Channels, height, width).\n        X = data[0].to(device) # X is the batch of features\n        # Unsqueeze adds a placeholder dimension for the color channel - (8, 28, 28) to (8, 1, 28, 28)\n        y = data[1].to(device) # y is the batch of targets.\n        \n        net.zero_grad()  # sets gradients to 0 before loss calc. You will do this likely every step.\n        output = net(X)  # pass in the reshaped batch (recall they are 28x28 atm)\n        tloss = criterion(output, y)  # calc and grab the loss value\n        tloss.backward()  # apply this loss backwards thru the network's parameters\n        optimizer.step()  # attempt to optimize weights to account for loss/gradients \n        \n        epoch_loss += tloss.item()\n        epoch_correct += get_num_correct(output, y)\n    print('LR =',scheduler.get_lr())    \n    scheduler.step() \n#     lr = optimizer.param_groups[0]['lr'] # ReduceLROnPlateau\n#     print('epoch',epoch,',lr',lr)\n#     scheduler.step(tloss)   \n\n    # Evaluation with the validation set\n    net.eval() # eval mode\n    val_loss = 0\n    val_correct = 0\n    test_loss = 0\n    test_correct = 0\n    \n    with torch.no_grad():\n        # First Validation Set\n        for data in val_loader:\n            X = data[0].to(device)\n            y = data[1].to(device)\n            \n            preds = net(X) # get predictions\n            vloss = criterion(preds, y) # calculate the loss\n            \n            val_correct += get_num_correct(preds, y)\n            val_loss += vloss.item()\n        \n        # Second Validation Set.. Dig-MNIST.csv\n        for data in test_loader:\n            X = data[0].to(device)\n            y = data[1].to(device)\n            \n            preds = net(X) # get predictions\n            tstloss = criterion(preds, y) # calculate the loss\n            \n            test_correct += get_num_correct(preds, y)\n            test_loss += tstloss.item()\n    \n    tmp_nn_output = [epoch + 1,EPOCHS,\n                     epoch_loss/len(train_loader.dataset),epoch_correct/len(train_loader.dataset)*100,\n                     val_loss/len(val_loader.dataset), val_correct/len(val_loader.dataset)*100,\n                     test_loss/len(test_loader.dataset), test_correct/len(test_loader.dataset)*100\n                    ]\n    nn_output.append(tmp_nn_output)\n    \n    # Print the loss and accuracy for the validation set\n    print('Epoch[{}/{}] Train loss: {:.6f} acc: {:.3f} | Valid loss: {:.6f} acc: {:.3f} | DigTest loss: {:.6f} acc: {:.3f}'\n        .format(*tmp_nn_output))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Result visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_results = pd.DataFrame(nn_output,\n    columns = ['epoch','total_epochs','train_loss','train_acc','valid_loss','valid_acc','test_loss','test_acc'])\n                         \ndisplay(pd_results)\n\nprint(\"Best Epoch: {}\".format(pd_results.loc[pd_results.valid_acc.idxmax()]['epoch']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nfig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(pd_results['epoch'],pd_results['valid_loss'], label='validation_loss')\naxes[0].plot(pd_results['epoch'],pd_results['train_loss'], label='train_loss')\n# axes[0].plot(pd_results['epoch'],pd_results['test_loss'], label='test_loss')\naxes[0].legend()\n\naxes[1].plot(pd_results['epoch'],pd_results['valid_acc'], label='validation_acc')\naxes[1].plot(pd_results['epoch'],pd_results['train_acc'], label='train_acc')\n# axes[1].plot(pd_results['epoch'],pd_results['test_acc'], label='test_acc')\naxes[1].legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Confusion Matrix"},{"metadata":{},"cell_type":"markdown","source":"val_loader = 0.2 * train.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(classes)\n\n# Use the validation set to make a confusion matrix\nnet.eval() # good habit I suppose\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Goes through the test set\nfor images, _ in val_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)\n\n# Make the confusion matrix\ncmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)\nfor i in range(len(val_labels)):\n    cmt[val_labels[i], predictions[i]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dig-MNIST.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(classes)\n\n# Use the validation set to make a confusion matrix\nnet.eval() # good habit I suppose\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Goes through the test set\nfor images, _ in test_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)\n\n# Make the confusion matrix\ncmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)\nfor i in range(len(test_labels)):\n    cmt[test_labels[i], predictions[i]] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Creat submission csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Time to get the network's predictions on the test set\n# Put the test set in a DataLoader\n\nnet.eval() # Safety first\npredictions = torch.LongTensor().to(device) # Tensor for all predictions\n\n# Go through the test set, saving the predictions in... 'predictions'\nfor images in submission_loader:\n    images = images.to(device)\n    preds = net(images)\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in the sample submission\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\n\n# Change the label column to our predictions \n# Have to make sure the predictions Tensor is on the cpu\nsubmission['label'] = predictions.cpu().numpy()\n# Write the dataframe to a new csv, not including the index\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}