{"cells":[{"metadata":{},"cell_type":"markdown","source":"+ private 리더보드 2등에 랭크한 inference kernel입니다.\n+ pytorch 기반 inference 커널이고 다른 대회 할 때도 참고하시면 도움될것 같습니다.\n+ 사실 순위 별로 기대 안하고 약 1달 전에 학습 그만둔 대회였으나 어찌하여 2등에 랭크하게 됐네요. 아마도 40개의 모델 (4fold tta2 + 8fold tta4을 앙상블 했기 때문에 robust한 결과를 얻을 수 있었던것 같습니다. (실전 대회는 inference 시간에 제한을 두기 때문에 웬만해서는 이렇게 하지 못합니다)\n+ 실전 대회였으면 아마 10 fold로 진행하였을것 같고, 많은 시도를 했을것 같습니다. ex) 서로 다른 이미지 사이즈 앙상블, focal loss와 같은 다른 criterion.\n+ public 리더보드가 10등 밖에 밀려났음에도 불구하고 최종 2등한걸 보면, 앙상블의 힘이 제대로 발휘된걸 알 수 있다고 생각합니다.\n+ senet142 4fold tta2 (imagenet pretrained, 워낙 커서 inference 하는데도 오래걸립니다) + resnext101_32x16d (instagram pretrained) 8fold를 앙상블 했습니다.\n\n+ 학습은 public에 공개한 커널 기반입니다. 특별한 trick은 안썼으며, 최대한 하이퍼 파라미터들을 고치면서 fitting 하도록 조절했습니다.\n+ 이번 대회를 통해서 많은 분들이 딥러닝 대회 리더보드에 나타나셨으면 좋겠습니다 :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport glob\nimport PIL\nfrom PIL import Image, ImageEnhance, ImageOps\nfrom collections import OrderedDict\nfrom joblib import Parallel, delayed\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport torch\nfrom torch import nn, cuda\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nimport torchvision.models as M\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 2019\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_boxing_img(img_name, margin=16) :\n    if img_name.split('_')[0] == \"train\" :\n        PATH = TRAIN_IMAGE_PATH\n        data = train_df\n    elif img_name.split('_')[0] == \"test\" :\n        PATH = TEST_IMAGE_PATH\n        data = test_df\n        \n    img = PIL.Image.open(os.path.join(PATH, img_name))\n    pos = data.loc[data[\"img_file\"] == img_name, \\\n                   ['bbox_x1','bbox_y1', 'bbox_x2', 'bbox_y2']].values.reshape(-1)\n\n    width, height = img.size\n    x1 = max(0, pos[0] - margin)\n    y1 = max(0, pos[1] - margin)\n    x2 = min(pos[2] + margin, width)\n    y2 = min(pos[3] + margin, height)\n\n    return img.crop((x1,y1,x2,y2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, test_imgs, transforms=None):\n        self.test_imgs = test_imgs\n        self.transform = transforms\n        \n    def __len__(self):\n        return len(self.test_imgs)\n    \n    def __getitem__(self, idx):\n        \n        image = self.test_imgs[idx].convert('RGB')\n            \n        if self.transform:\n            image = self.transform(image)\n            \n        return image        ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/2019-3rd-ml-month-with-kakr/train.csv')\nDATA_PATH = '../input/2019-3rd-ml-month-with-kakr'\nTEST_IMAGE_PATH = os.path.join(DATA_PATH, 'test')\ntest_df = pd.read_csv('../input/2019-3rd-ml-month-with-kakr/test.csv')\nnum_classes = df['class'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nx_test = Parallel(n_jobs=4)(\n    delayed(lambda x: crop_boxing_img(x))(x) for x in tqdm(test_df['img_file']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Resnet50"},{"metadata":{"trusted":true},"cell_type":"code","source":"# target_size = (224, 224)\n\n# data_transforms = transforms.Compose([\n#         transforms.Resize(target_size),\n#         transforms.RandomResizedCrop(target_size, scale=(0.8,1.0)),\n#         transforms.RandomHorizontalFlip(),\n#         transforms.ToTensor(),\n#         transforms.Normalize(\n#             [0.485, 0.456, 0.406], \n#             [0.229, 0.224, 0.225])\n#     ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resnet50_weights_path = Path('../input/car-4fold-weights/')\n# weight_list = os.listdir(resnet50_weights_path)\n# weight_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n\n# batch_size = 1\n# tta = 5\n# test_dataset = TestDataset(x_test, transforms=data_transforms)\n# test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n# total_num_models = len(weight_list)*tta \n\n# model = M.resnet50() \n# model.fc = nn.Linear(2048, num_classes)\n# model.cuda()\n\n# resnet50_prediction = np.zeros((len(test_dataset), num_classes))\n\n# for i, weight in enumerate(weight_list):\n#     print(\"weight {} prediction starts\".format(i+1))\n    \n#     for _ in range(tta):\n#         print(\"tta {}\".format(_+1))\n\n#         model.load_state_dict(torch.load(resnet50_weights_path / weight))\n\n#         model.eval()\n        \n#         prediction = np.zeros((len(test_dataset), num_classes))\n#         with torch.no_grad():\n#             for i, images in enumerate(test_loader):\n#                 images = images.cuda()\n\n#                 preds = model(images).detach()\n#                 prediction[i * batch_size: (i+1) * batch_size] = preds.cpu().numpy()\n#                 resnet50_prediction = resnet50_prediction + prediction\n#         del prediction\n        \n# resnet50_prediction /= total_num_models\n\n# del test_dataset\n# del test_loader\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Bottleneck(nn.Module):\n    \"\"\"\n    Base class for bottlenecks that implements `forward()` method.\n    \"\"\"\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n\n        return out\n\nclass SEBottleneck(Bottleneck):\n    \"\"\"\n    Bottleneck for SENet154.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,\n                               stride=stride, padding=1, groups=groups,\n                               bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,\n                               bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n        \n        \nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        \"\"\"\n        Parameters\n        ----------\n        block (nn.Module): Bottleneck class.\n            - For SENet154: SEBottleneck\n            - For SE-ResNet models: SEResNetBottleneck\n            - For SE-ResNeXt models:  SEResNeXtBottleneck\n        layers (list of ints): Number of residual blocks for 4 layers of the\n            network (layer1...layer4).\n        groups (int): Number of groups for the 3x3 convolution in each\n            bottleneck block.\n            - For SENet154: 64\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models:  32\n        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.\n            - For all models: 16\n        dropout_p (float or None): Drop probability for the Dropout layer.\n            If `None` the Dropout layer is not used.\n            - For SENet154: 0.2\n            - For SE-ResNet models: None\n            - For SE-ResNeXt models: None\n        inplanes (int):  Number of input channels for layer1.\n            - For SENet154: 128\n            - For SE-ResNet models: 64\n            - For SE-ResNeXt models: 64\n        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of\n            a single 7x7 convolution in layer0.\n            - For SENet154: True\n            - For SE-ResNet models: False\n            - For SE-ResNeXt models: False\n        downsample_kernel_size (int): Kernel size for downsampling convolutions\n            in layer2, layer3 and layer4.\n            - For SENet154: 3\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models: 1\n        downsample_padding (int): Padding for downsampling convolutions in\n            layer2, layer3 and layer4.\n            - For SENet154: 1\n            - For SE-ResNet models: 0\n            - For SE-ResNeXt models: 0\n        num_classes (int): Number of outputs in `last_linear` layer.\n            - For all models: 1000\n        \"\"\"\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,\n                                    bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,\n                                    padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        # To preserve compatibility with Caffe weights `ceil_mode=True`\n        # is used instead of `padding=1`.\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,\n                                                    ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.avg_pool = nn.AvgPool2d(7, stride=1)\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,\n                    downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=downsample_kernel_size, stride=stride,\n                          padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride,\n                            downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\n\ndef senet154(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,\n                  dropout_p=0.2, num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['senet154'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Senet154"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_size = (224, 224)\n\ndata_transforms = transforms.Compose([\n            transforms.Resize(target_size),\n            transforms.RandomHorizontalFlip(),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                [0.485, 0.456, 0.406], \n                [0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"senet154_weights_path = Path('../input/car-senet142/')\nweight_list = os.listdir(senet154_weights_path)\nweight_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nbatch_size = 1\ntta = 2\ntest_dataset = TestDataset(x_test, transforms=data_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\ntotal_num_models = len(weight_list)*tta \n\nmodel = senet154(num_classes=1000, pretrained=None)\nmodel.last_linear = nn.Linear(model.last_linear.in_features, num_classes)\nmodel.cuda()\n\nsenet154_prediction = np.zeros((len(test_dataset), num_classes))\n\nfor i, weight in enumerate(weight_list):\n    print(\"weight {} prediction starts\".format(i+1))\n    \n    for _ in range(tta):\n        print(\"tta {}\".format(_+1))\n        model.load_state_dict(torch.load(senet154_weights_path / weight))\n\n        model.eval()\n        \n        prediction = np.zeros((len(test_dataset), num_classes))\n        with torch.no_grad():\n            for i, images in enumerate(test_loader):\n                images = images.cuda()\n\n                preds = model(images).detach()\n                prediction[i * batch_size: (i+1) * batch_size] = preds.cpu().numpy()\n                senet154_prediction = senet154_prediction + prediction\n        del prediction\n    \n    \nsenet154_prediction /= total_num_models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnext_weights_path = Path('../input/car-resnext-weights/')\nweight_list = os.listdir(resnext_weights_path)\nweight_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nbatch_size = 1\ntta = 4\ntotal_num_models = len(weight_list)*tta \n\nmodel = torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')\nmodel.fc = nn.Sequential(nn.Dropout(0.2), nn.Linear(2048, num_classes))\nmodel.cuda()\n\nresnext_prediction = np.zeros((len(test_dataset), num_classes))\n\nfor i, weight in enumerate(weight_list):\n    print(\"weight {} prediction starts\".format(i+1))\n    \n    for _ in range(tta):\n        print(\"tta {}\".format(_+1))\n        model.load_state_dict(torch.load(resnext_weights_path / weight))\n\n        model.eval()\n        \n        prediction = np.zeros((len(test_dataset), num_classes))\n        with torch.no_grad():\n            for i, images in enumerate(test_loader):\n                images = images.cuda()\n\n                preds = model(images).detach()\n                prediction[i * batch_size: (i+1) * batch_size] = preds.cpu().numpy()\n                resnext_prediction = resnext_prediction + prediction\n        del prediction\n        \n    \nresnext_prediction /= total_num_models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_prediction = (senet154_prediction/2) + (resnext_prediction/2)\n# all_prediction = (resnet50_prediction/3) + (resnext_prediction/3) + (senet154_prediction/3)\n\nresult = np.argmax(all_prediction, axis=1)\nsubmission = pd.read_csv('../input/2019-3rd-ml-month-with-kakr/sample_submission.csv')\nsubmission[\"class\"] = result\nsubmission[\"class\"].replace(0, 196, inplace=True)\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}