{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":29193,"databundleVersionId":2318453,"sourceType":"competition"}],"dockerImageVersionId":30096,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm\n!pip install albumentations==1.0.2\n!pip install opencv_python==4.5.2.52","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:45:41.587626Z","iopub.execute_input":"2021-07-15T23:45:41.587952Z","iopub.status.idle":"2021-07-15T23:46:06.791422Z","shell.execute_reply.started":"2021-07-15T23:45:41.587922Z","shell.execute_reply":"2021-07-15T23:46:06.790454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport cv2\nfrom tqdm import tqdm\nimport numpy as np\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import StepLR, CosineAnnealingLR\nfrom sklearn.model_selection import RepeatedKFold, StratifiedKFold\nimport pandas as pd\n\n# Augmentation\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n# model\nimport timm\n\n# os.mkdir('classify-leaves')\n\nif torch.cuda.is_available():\n    device = torch.device('cuda')\nelse:\n    device = torch.device('cpu')\n\nprint(f'Using device: {device}')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T08:17:38.577456Z","iopub.execute_input":"2021-07-16T08:17:38.577772Z","iopub.status.idle":"2021-07-16T08:17:38.584989Z","shell.execute_reply.started":"2021-07-16T08:17:38.577742Z","shell.execute_reply":"2021-07-16T08:17:38.584113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCH = 36\nLR = 3e-4\nBATCH_SIZE = 64\nN_FOLD = 5\nNUM_CLASS = 176\nIMG_SIZE = 224\nDECAY_SCALE = 20.0","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:46:07.697046Z","iopub.execute_input":"2021-07-15T23:46:07.697859Z","iopub.status.idle":"2021-07-15T23:46:07.70351Z","shell.execute_reply.started":"2021-07-15T23:46:07.697807Z","shell.execute_reply":"2021-07-15T23:46:07.702425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DATA_PATH = '../input/classify-leaves/test.csv'\nTRAIN_DATA_PATH = '../input/classify-leaves/train.csv'\n\nlabels = pd.read_csv(TRAIN_DATA_PATH)['label'].values\nlabel_list = sorted(list(set(labels)))\nn_classes = len(label_list)\n\nclass_to_num = dict(zip(label_list, range(n_classes)))\nnum_to_class = {v: k for k, v in class_to_num.items()}\n\ntrain_df = pd.read_csv(TRAIN_DATA_PATH)\nlabel = train_df['label'].values\ntrain_df['label'] = [class_to_num[i] for i in label]\n\ntest_df = pd.read_csv(TEST_DATA_PATH)\n\nprint(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:46:07.705405Z","iopub.execute_input":"2021-07-15T23:46:07.706042Z","iopub.status.idle":"2021-07-15T23:46:07.799449Z","shell.execute_reply.started":"2021-07-15T23:46:07.705997Z","shell.execute_reply":"2021-07-15T23:46:07.7985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataSet","metadata":{}},{"cell_type":"code","source":"def randAugment(N=2, M=4, p=1.0, mode=\"all\", cut_out=False):\n    \"\"\"\n    Examples:\n        >>> # M from 0 to 20\n        >>> transforms = randAugment(N=3, M=8, p=0.8, mode='all', cut_out=False)\n    \"\"\"\n    # Magnitude(M) search space\n    scale = np.linspace(0, 0.4, 20)\n    translate = np.linspace(0, 0.4, 20)\n    rot = np.linspace(0, 30, 20)\n    shear_x = np.linspace(0, 20, 20)\n    shear_y = np.linspace(0, 20, 20)\n    contrast = np.linspace(0.0, 0.4, 20)\n    bright = np.linspace(0.0, 0.4, 20)\n    sat = np.linspace(0.0, 0.2, 20)\n    hue = np.linspace(0.0, 0.2, 20)\n    shar = np.linspace(0.0, 0.9, 20)\n    blur = np.linspace(0, 0.2, 20)\n    noise = np.linspace(0, 1, 20)\n    cut = np.linspace(0, 0.6, 20)\n    # Transformation search space\n    Aug = [  # geometrical\n        albumentations.Affine(scale=(1.0 - scale[M], 1.0 + scale[M]), p=p),\n        albumentations.Affine(translate_percent=(-translate[M], translate[M]), p=p),\n        albumentations.Affine(rotate=(-rot[M], rot[M]), p=p),\n        albumentations.Affine(shear={'x': (-shear_x[M], shear_x[M])}, p=p),\n        albumentations.Affine(shear={'y': (-shear_y[M], shear_y[M])}, p=p),\n        # Color Based\n        albumentations.RandomBrightnessContrast(brightness_limit=bright[M], contrast_limit=contrast[M], p=p),\n        albumentations.ColorJitter(brightness=0.0, contrast=0.0, saturation=sat[M], hue=0.0, p=p),\n        albumentations.ColorJitter(brightness=0.0, contrast=0.0, saturation=0.0, hue=hue[M], p=p),\n        albumentations.Sharpen(alpha=(0.1, shar[M]), lightness=(0.5, 1.0), p=p),\n        albumentations.core.composition.PerChannel(\n            albumentations.OneOf([\n                albumentations.MotionBlur(p=0.5),\n                albumentations.MedianBlur(blur_limit=3, p=1),\n                albumentations.Blur(blur_limit=3, p=1), ])\n            , p=blur[M] * p),\n        albumentations.GaussNoise(var_limit=(8.0 * noise[M], 64.0 * noise[M]), per_channel=True, p=p)\n    ]\n    # Sampling from the Transformation search space\n    if mode == \"geo\":\n        transforms = albumentations.SomeOf(Aug[0:5], N)\n    elif mode == \"color\":\n        transforms = albumentations.SomeOf(Aug[5:], N)\n    else:\n        transforms = albumentations.SomeOf(Aug, N)\n\n    if cut_out:\n        cut_trans = albumentations.OneOf([\n            albumentations.CoarseDropout(max_holes=8, max_height=16, max_width=16, fill_value=0, p=1),\n            albumentations.GridDropout(ratio=cut[M], p=1),\n            albumentations.Cutout(num_holes=8, max_h_size=16, max_w_size=16, p=1),\n        ], p=cut[M])\n        transforms = albumentations.Compose([transforms, cut_trans])\n\n    return transforms\n\n\nRESIZE_SIZE = 224\n\ntrain_preprocess = albumentations.Compose([\n    albumentations.Resize(RESIZE_SIZE, RESIZE_SIZE),\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.0625, rotate_limit=45, border_mode=1, p=0.5),\n    randAugment(),\n    albumentations.Normalize(),\n    ToTensorV2()\n])\n\ntest_preprocess = albumentations.Compose([\n    albumentations.Resize(RESIZE_SIZE, RESIZE_SIZE),\n    albumentations.Normalize(),\n    ToTensorV2()\n])\n\ndef train_loader(path):\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = train_preprocess(image=img)['image']\n    return img\n\n\ndef test_loader(path):\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = test_preprocess(image=img)['image']\n    return img\n\n\nclass ImageDataSet(Dataset):\n    def __init__(self, df, data_path='../input/classify-leaves', train=True, tta=False):\n        if train or tta:\n            self.loader = train_loader\n        else:\n            self.loader = test_loader\n        self.df = df\n        self.data_path = data_path\n\n    def __getitem__(self, index):\n        img_path = self.df.image[index]\n        img_path = os.path.join(self.data_path, img_path)\n        img = self.loader(img_path)\n\n        if 'label' in self.df.columns:\n            label = self.df.label[index]\n            return img, label\n\n        return img\n\n    def __len__(self):\n        return self.df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:46:07.800838Z","iopub.execute_input":"2021-07-15T23:46:07.801236Z","iopub.status.idle":"2021-07-15T23:46:07.829663Z","shell.execute_reply.started":"2021-07-15T23:46:07.801195Z","shell.execute_reply":"2021-07-15T23:46:07.828723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\n\nclass FastGlobalAvgPool2d(nn.Module):\n    def __init__(self, flatten=True): # flatten == True : pool + flatten\n        super(FastGlobalAvgPool2d, self).__init__()\n        self.flatten = flatten\n\n    def forward(self, x):\n        if self.flatten:\n            in_size = x.size()\n            return x.view((in_size[0], in_size[1], -1)).mean(dim=2)\n        else:\n            return x.view(x.size(0), x.size(1), -1).mean(-1).view(x.size(0), x.size(1), 1, 1)\n\n\nclass FastGlobalConcatPool2d(nn.Module):\n    def __init__(self, flatten=True): # flatten == True : pool + flatten\n        super(FastGlobalConcatPool2d, self).__init__()\n        self.flatten = flatten\n\n    def forward(self, x):\n        if self.flatten:\n            in_size = x.size()\n            x = x.view((in_size[0], in_size[1], -1))\n            return torch.cat([x.mean(dim=2), x.max(dim=2).values], 1)\n        else:\n            x = x.view(x.size(0), x.size(1), -1)\n            return torch.cat([x.mean(-1), x.max(-1).values], 1).view(x.size(0), 2*x.size(1), 1, 1)\n\nclass MultiSampleDropoutFC(nn.Module):\n    def __init__(self, in_ch, out_ch, num_sample = 5, dropout = 0.5):\n        super(MultiSampleDropoutFC, self).__init__()\n        self.dropouts = nn.ModuleList([nn.Dropout(dropout) for _ in range(num_sample)])\n        self.fc = nn.Linear(in_ch, out_ch, bias=True)\n\n    def forward(self,x):\n        out = None\n        for i, dropout in enumerate(self.dropouts):\n            if i == 0:\n                out = self.fc(dropout(x))\n            else:\n                out += self.fc(dropout(x))\n        out /= len(self.dropouts)\n        return out\n\ndef create_model(name, pretrained, pool=True):\n    if pool:    # pool with flatten (bs, c)\n        model = timm.create_model(name, features_only=True, pretrained=pretrained)\n    else:       # (bs, c, f_h, f_w)\n        model = timm.create_model(name, features_only=True, pretrained=pretrained,\n                                      global_pool='', num_classes=0)\n    return model\n\n\nclass ImageModel(nn.Module):\n\n    def __init__(self, name='resnest50d', pretrained=True, pooling='concat', fc='multi-dropout', feature=2048,\n                 classes=0):\n        super(ImageModel, self).__init__()\n        self.model = create_model(name, pretrained, pool=False)\n\n        if pooling == 'concat':\n            self.pooling = FastGlobalConcatPool2d()\n            feature *= 2\n        else:\n            self.pooling = FastGlobalAvgPool2d()\n\n        if fc == 'multi-dropout':\n            self.fc = nn.Sequential(\n                MultiSampleDropoutFC(in_ch=feature, out_ch=classes)\n            )\n        elif fc == '2layers':\n            self.fc = nn.Sequential(\n                nn.Linear(feature, 512, bias=False),\n                nn.BatchNorm1d(512),\n                torch.nn.SiLU(inplace=True),\n                nn.Dropout(),\n                nn.Linear(512, classes, bias=True),\n            )\n        else:\n            self.fc = nn.Linear(in_features=feature, out_features=classes, bias=True)\n\n    def forward(self, x):\n        feature_map = self.model(x)[-1]\n        embedding = self.pooling(feature_map)\n        logits = self.fc(embedding)\n        return logits, embedding\n","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:46:07.831219Z","iopub.execute_input":"2021-07-15T23:46:07.83159Z","iopub.status.idle":"2021-07-15T23:46:07.854969Z","shell.execute_reply.started":"2021-07-15T23:46:07.831555Z","shell.execute_reply":"2021-07-15T23:46:07.853979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Optim Ranger","metadata":{}},{"cell_type":"code","source":"# Ranger deep learning optimizer - RAdam + Lookahead + Gradient Centralization, combined into one optimizer.\n# https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer\n\nimport math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\n\ndef centralized_gradient(x, use_gc=True, gc_conv_only=False):\n    \"\"\"credit - https://github.com/Yonghongwei/Gradient-Centralization \"\"\"\n    if use_gc:\n        if gc_conv_only:\n            if len(list(x.size())) > 3:\n                x.add_(-x.mean(dim=tuple(range(1, len(list(x.size())))), keepdim=True))\n        else:\n            if len(list(x.size())) > 1:\n                x.add_(-x.mean(dim=tuple(range(1, len(list(x.size())))), keepdim=True))\n    return x\n\n\nclass Ranger(Optimizer):\n\n    def __init__(self, params, lr=1e-3,                       # lr\n                 alpha=0.5, k=6, N_sma_threshhold=5,           # Ranger options\n                 betas=(.95, 0.999), eps=1e-5, weight_decay=0,  # Adam options\n                 # Gradient centralization on or off, applied to conv layers only or conv + fc layers\n                 use_gc=True, gc_conv_only=False, gc_loc=True\n                 ):\n\n        # parameter checks\n        if not 0.0 <= alpha <= 1.0:\n            raise ValueError(f'Invalid slow update rate: {alpha}')\n        if not 1 <= k:\n            raise ValueError(f'Invalid lookahead steps: {k}')\n        if not lr > 0:\n            raise ValueError(f'Invalid Learning Rate: {lr}')\n        if not eps > 0:\n            raise ValueError(f'Invalid eps: {eps}')\n\n        # parameter comments:\n        # beta1 (momentum) of .95 seems to work better than .90...\n        # N_sma_threshold of 5 seems better in testing than 4.\n        # In both cases, worth testing on your dataset (.90 vs .95, 4 vs 5) to make sure which works best for you.\n\n        # prep defaults and init torch.optim base\n        defaults = dict(lr=lr, alpha=alpha, k=k, step_counter=0, betas=betas,\n                        N_sma_threshhold=N_sma_threshhold, eps=eps, weight_decay=weight_decay)\n        super().__init__(params, defaults)\n\n        # adjustable threshold\n        self.N_sma_threshhold = N_sma_threshhold\n\n        # look ahead params\n\n        self.alpha = alpha\n        self.k = k\n\n        # radam buffer for state\n        self.radam_buffer = [[None, None, None] for ind in range(10)]\n\n        # gc on or off\n        self.gc_loc = gc_loc\n        self.use_gc = use_gc\n        self.gc_conv_only = gc_conv_only\n        # level of gradient centralization\n        #self.gc_gradient_threshold = 3 if gc_conv_only else 1\n\n        print(\n            f\"Ranger optimizer loaded. \\nGradient Centralization usage = {self.use_gc}\")\n        if self.use_gc and self.gc_conv_only == False:\n            print(f\"GC applied to both conv and fc layers\")\n        elif self.use_gc and self.gc_conv_only == True:\n            print(f\"GC applied to conv layers only\")\n\n    def __setstate__(self, state):\n        print(\"set state called\")\n        super(Ranger, self).__setstate__(state)\n\n    def step(self, closure=None):\n        loss = None\n        # note - below is commented out b/c I have other work that passes back the loss as a float, and thus not a callable closure.\n        # Uncomment if you need to use the actual closure...\n\n        # if closure is not None:\n        #loss = closure()\n\n        # Evaluate averages and grad, update param tensors\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n\n                if grad.is_sparse:\n                    raise RuntimeError(\n                        'Ranger optimizer does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]  # get state dict for this param\n\n                if len(state) == 0:  # if first time to run...init dictionary with our desired entries\n                    # if self.first_run_check==0:\n                    # self.first_run_check=1\n                    #print(\"Initializing slow buffer...should not see this at load from saved model!\")\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n\n                    # look ahead weight storage now in state dict\n                    state['slow_buffer'] = torch.empty_like(p.data)\n                    state['slow_buffer'].copy_(p.data)\n\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(\n                        p_data_fp32)\n\n                # begin computations\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                # GC operation for Conv layers and FC layers\n                # if grad.dim() > self.gc_gradient_threshold:\n                #    grad.add_(-grad.mean(dim=tuple(range(1, grad.dim())), keepdim=True))\n                if self.gc_loc:\n                    grad = centralized_gradient(grad, use_gc=self.use_gc, gc_conv_only=self.gc_conv_only)\n\n                state['step'] += 1\n\n                # compute variance mov avg\n                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n\n                # compute mean moving avg\n                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n\n                buffered = self.radam_buffer[int(state['step'] % 10)]\n\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * \\\n                        state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n                    if N_sma > self.N_sma_threshhold:\n                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (\n                            N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = 1.0 / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                # if group['weight_decay'] != 0:\n                #    p_data_fp32.add_(-group['weight_decay']\n                #                     * group['lr'], p_data_fp32)\n\n                # apply lr\n                if N_sma > self.N_sma_threshhold:\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    G_grad = exp_avg / denom\n                else:\n                    G_grad = exp_avg\n\n                if group['weight_decay'] != 0:\n                    G_grad.add_(p_data_fp32, alpha=group['weight_decay'])\n                # GC operation\n                if not self.gc_loc:\n                    G_grad = centralized_gradient(G_grad, use_gc=self.use_gc, gc_conv_only=self.gc_conv_only)\n\n                p_data_fp32.add_(G_grad, alpha=-step_size * group['lr'])\n                p.data.copy_(p_data_fp32)\n\n                # integrated look ahead...\n                # we do it at the param level instead of group level\n                if state['step'] % group['k'] == 0:\n                    # get access to slow param tensor\n                    slow_p = state['slow_buffer']\n                    # (fast weights - slow weights) * alpha\n                    slow_p.add_(p.data - slow_p, alpha=self.alpha)\n                    # copy interpolated weights to RAdam param tensor\n                    p.data.copy_(slow_p)\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:46:07.856528Z","iopub.execute_input":"2021-07-15T23:46:07.856909Z","iopub.status.idle":"2021-07-15T23:46:07.88906Z","shell.execute_reply.started":"2021-07-15T23:46:07.856873Z","shell.execute_reply":"2021-07-15T23:46:07.888045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CELOSS","metadata":{}},{"cell_type":"code","source":"import torch.nn.functional as F\nfrom torch.nn.modules.loss import _WeightedLoss\n\n\nclass LabelSmoothingCrossEntropy(nn.Module):\n    \"\"\"\n    NLL loss with label smoothing.\n    \"\"\"\n\n    def __init__(self, smoothing=0.1):\n        \"\"\"\n        Constructor for the LabelSmoothing module.\n        :param smoothing: label smoothing factor\n        \"\"\"\n        super(LabelSmoothingCrossEntropy, self).__init__()\n        assert smoothing < 1.0\n        self.smoothing = smoothing\n        self.confidence = 1. - smoothing\n\n    def forward(self, x, target):\n        logprobs = F.log_softmax(x, dim=-1)\n        nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))\n        nll_loss = nll_loss.squeeze(1)\n        smooth_loss = -logprobs.mean(dim=-1)\n        loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n        return loss.mean()\n\n\nclass SmoothBCEwLogits(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean', smoothing=0.0):\n        super().__init__(weight=weight, reduction=reduction)\n        self.smoothing = smoothing\n        self.weight = weight\n        self.reduction = reduction\n\n    @staticmethod\n    def _smooth(targets: torch.Tensor, n_labels: int, smoothing=0.0):\n        assert 0 <= smoothing < 1\n        with torch.no_grad():\n            targets = targets * (1.0 - smoothing) + 0.5 * smoothing\n        return targets\n\n    def forward(self, inputs, targets):\n        targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1), self.smoothing)\n        loss = F.binary_cross_entropy_with_logits(inputs, targets, self.weight)\n\n        if self.reduction == 'sum':\n            loss = loss.sum()\n        elif self.reduction == 'mean':\n            loss = loss.mean()\n        else:\n            loss = loss.mean()\n\n        return loss\n\n\nclass SoftTargetCrossEntropy(nn.Module):\n\n    def __init__(self):\n        super(SoftTargetCrossEntropy, self).__init__()\n\n    def forward(self, x, target):\n        loss = torch.sum(-target * F.log_softmax(x, dim=-1), dim=-1)\n        return loss.mean()\n\n\nclass KLDivLoss(nn.Module):\n    \"\"\"KL-divergence with softmax\"\"\"\n\n    def __init__(self):\n        super(KLDivLoss, self).__init__()\n        self.loss = nn.KLDivLoss()\n\n    def forward(self, model_output, target):\n        log = F.log_softmax(model_output, dim=1)\n        loss = self.loss(log, target)\n        return loss\n\n\nclass topkLoss(nn.Module):\n    \"\"\"topkLoss: Online Hard Example Mining\"\"\"\n\n    def __init__(self, loss, top_k=0.75):\n        super(topkLoss, self).__init__()\n        self.top_k = top_k\n        self.loss = loss\n\n    def forward(self, input, target):\n        loss = self.loss(input, target)\n        if self.top_k == 1:\n            return torch.mean(loss)\n        else:\n            valid_loss, idxs = torch.topk(loss, round(self.top_k * loss.size()[0]), dim=0)\n            return torch.mean(valid_loss)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-15T23:46:07.891728Z","iopub.execute_input":"2021-07-15T23:46:07.892348Z","iopub.status.idle":"2021-07-15T23:46:07.913875Z","shell.execute_reply.started":"2021-07-15T23:46:07.892308Z","shell.execute_reply":"2021-07-15T23:46:07.912476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Focal Loss ","metadata":{}},{"cell_type":"code","source":"class MultiFocalLoss(nn.Module):\n    \"\"\"\n    This is a implementation of Focal Loss with smooth label cross entropy supported which is proposed in\n    'Focal Loss for Dense Object Detection. (https://arxiv.org/abs/1708.02002)'\n        Focal_Loss= -1*alpha*(1-pt)^gamma*log(pt)\n    :param num_class:\n    :param alpha: (tensor) 3D or 4D the scalar factor for this criterion\n    :param gamma: (float,double) gamma > 0 reduces the relative loss for well-classified examples (p>0.5) putting more\n                    focus on hard misclassified example\n    :param smooth: (float,double) smooth value when cross entropy\n    :param balance_index: (int) balance class index, should be specific when alpha is float\n    :param size_average: (bool, optional) By default, the losses are averaged over each loss element in the batch.\n    \"\"\"\n\n    def __init__(self, num_class, alpha=None, gamma=2, balance_index=-1, smooth=None, size_average=True):\n        super(MultiFocalLoss, self).__init__()\n        self.num_class = num_class\n        self.alpha = alpha\n        self.gamma = gamma\n        self.smooth = smooth\n        self.size_average = size_average\n\n        if self.alpha is None:\n            self.alpha = torch.ones(self.num_class, 1)\n        elif isinstance(self.alpha, (list, np.ndarray)):\n            assert len(self.alpha) == self.num_class\n            self.alpha = torch.FloatTensor(alpha).view(self.num_class, 1)\n            self.alpha = self.alpha / self.alpha.sum()\n        elif isinstance(self.alpha, float):\n            alpha = torch.ones(self.num_class, 1)\n            alpha = alpha * (1 - self.alpha)\n            alpha[balance_index] = self.alpha\n            self.alpha = alpha\n        else:\n            raise TypeError('Not support alpha type')\n\n        if self.smooth is not None:\n            if self.smooth < 0 or self.smooth > 1.0:\n                raise ValueError('smooth value should be in [0,1]')\n\n    def forward(self, input, target):\n        logit = F.softmax(input, dim=1)\n\n        if logit.dim() > 2:\n            # N,C,d1,d2 -> N,C,m (m=d1*d2*...)\n            logit = logit.view(logit.size(0), logit.size(1), -1)\n            logit = logit.permute(0, 2, 1).contiguous()\n            logit = logit.view(-1, logit.size(-1))\n        target = target.view(-1, 1)\n\n        # N = input.size(0)\n        # alpha = torch.ones(N, self.num_class)\n        # alpha = alpha * (1 - self.alpha)\n        # alpha = alpha.scatter_(1, target.long(), self.alpha)\n        epsilon = 1e-10\n        alpha = self.alpha\n        if alpha.device != input.device:\n            alpha = alpha.to(input.device)\n\n        idx = target.cpu().long()\n        one_hot_key = torch.FloatTensor(target.size(0), self.num_class).zero_()\n        one_hot_key = one_hot_key.scatter_(1, idx, 1)\n        if one_hot_key.device != logit.device:\n            one_hot_key = one_hot_key.to(logit.device)\n\n        if self.smooth:\n            one_hot_key = torch.clamp(\n                one_hot_key, self.smooth, 1.0 - self.smooth)\n        pt = (one_hot_key * logit).sum(1) + epsilon\n        logpt = pt.log()\n\n        gamma = self.gamma\n\n        alpha = alpha[idx]\n        loss = -1 * alpha * torch.pow((1 - pt), gamma) * logpt\n\n        if self.size_average:\n            loss = loss.mean()\n        else:\n            loss = loss.sum()\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-07-16T00:54:24.78196Z","iopub.execute_input":"2021-07-16T00:54:24.782306Z","iopub.status.idle":"2021-07-16T00:54:24.798554Z","shell.execute_reply.started":"2021-07-16T00:54:24.78227Z","shell.execute_reply":"2021-07-16T00:54:24.797346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train and Validation","metadata":{}},{"cell_type":"code","source":"splitter = StratifiedKFold(n_splits=N_FOLD, shuffle=False)\n\ntr_folds = []\nval_folds = []\nfor train_idx, val_idx in splitter.split(train_df.image, train_df.label):\n    tr_folds.append(train_idx)\n    val_folds.append(val_idx)\n\ndef train_model():\n    tr_loss = []\n    tr_acc = []\n    model_conv.train()\n    for tr_imgs, tr_labels in tqdm(tr_data):\n        tr_imgs = tr_imgs.to(device)\n        tr_labels = tr_labels.to(device)\n        predicts, _ = model_conv(tr_imgs)\n        loss = criterion(predicts, tr_labels)\n        optim.zero_grad()\n        loss.backward()\n        optim.step()\n        scheduler.step()\n\n        tr_acc.append((predicts.argmax(dim=-1) == tr_labels).float().mean())\n        tr_loss.append(loss.data.cpu().numpy())\n\n    return sum(tr_loss) / len(tr_loss), sum(tr_acc) / len(tr_acc)\n\n\ndef val_model():\n    val_loss = []\n    val_acc = []\n    model_conv.eval()\n    for val_imgs, val_labels in tqdm(val_data):\n        val_imgs = val_imgs.to(device)\n        val_labels = val_labels.to(device)\n        with torch.no_grad():\n            predicts, _ = model_conv(val_imgs)\n        val_loss.append(criterion_test(predicts, val_labels).data.cpu().numpy())\n        val_acc.append((predicts.argmax(dim=-1) == val_labels).float().mean())\n\n    return sum(val_loss)/len(val_loss), sum(val_acc)/len(val_acc)\n\n\ndef train():\n    best_loss = 100.0\n    best_acc = 0.0\n    avg_val_loss, avg_val_acc = val_model()\n    print('pretrain val loss %.4f precision %.4f' % (avg_val_loss, avg_val_acc))\n\n    for epoch in range(EPOCH):\n        avg_tr_loss, avg_tr_acc = train_model()\n        avg_val_loss, avg_val_acc = val_model()\n\n        print(f\"[ Epoch | {epoch + 1:02d}/{EPOCH:02d} ] lr = {optim.param_groups[0]['lr']:.5f} |\"\n              f\" trn_loss = {avg_tr_loss:.5f} |\"\n              f\" trn_acc = {avg_tr_acc:.5f} |\"\n              f\" val_loss = {avg_val_loss:.5f} |\"\n              f\" val_acc = {avg_val_acc:.5f}\")\n\n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n\n        if avg_val_acc > best_acc:\n            best_acc = avg_val_acc\n            torch.save(model_conv.state_dict(), model_save_path)\n            print(f'saving model with acc {best_acc:.5f}')\n\n    return best_loss, best_acc","metadata":{"execution":{"iopub.status.busy":"2021-07-16T07:40:37.544201Z","iopub.execute_input":"2021-07-16T07:40:37.544554Z","iopub.status.idle":"2021-07-16T07:40:37.569482Z","shell.execute_reply.started":"2021-07-16T07:40:37.544512Z","shell.execute_reply":"2021-07-16T07:40:37.568564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_models = {\n    'resnest50d': 2048,\n    'resnetv2_50x1_bitm': 2048,\n    'swsl_resnext50_32x4d': 2048,\n    'densenet121': 1024,\n    'seresnext50_32x4d': 2048,\n}\n\ncv_losses = []\ncv_metrics = []\n\nfor fold in range(N_FOLD):\n    print(f'\\n ********** Fold {fold+1:02d} **********\\n')\n\n    ####################### Data ########################\n    tr_set = ImageDataSet(df=train_df.iloc[tr_folds[fold]].reset_index())\n    tr_data = DataLoader(tr_set, batch_size=BATCH_SIZE, shuffle=True)\n\n    val_set = ImageDataSet(df=train_df.iloc[val_folds[fold]].reset_index(), train=False)\n    val_data = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=True)\n\n    ####################### Model ########################\n    tup = conv_models.popitem()\n\n    model_conv = ImageModel(name=tup[0], pretrained=True, feature=tup[1], classes=NUM_CLASS)\n    model_conv = nn.DataParallel(model_conv)\n    model_conv = model_conv.to(device)\n\n    optim = Ranger(model_conv.parameters(), lr=LR, weight_decay=2e-4)\n    \n    model_save_path = f'./classify-leaves/model_{tup[0]}.pth'\n\n    criterion = MultiFocalLoss(num_class=NUM_CLASS, alpha=0.25)\n#     criterion = LabelSmoothingCrossEntropy().to(device)\n\n    criterion_test = nn.CrossEntropyLoss().to(device)\n\n    T = len(tr_data) * EPOCH # cycle\n    scheduler = CosineAnnealingLR(optim, T_max=T, eta_min=LR / DECAY_SCALE)\n\n    ####################### Train ########################\n    if fold < 2:\n        continue\n    _loss, _acc = train()\n\n    cv_losses.append(_loss)\n    cv_metrics.append(_acc)\n       \ncv_loss = sum(cv_losses) / N_FOLD\ncv_acc = sum(cv_metrics) / N_FOLD\nprint('CV loss:%.6f  CV precision:%.6f'%(cv_loss, cv_acc))\n","metadata":{"execution":{"iopub.status.busy":"2021-07-16T08:24:46.689667Z","iopub.execute_input":"2021-07-16T08:24:46.689991Z"},"trusted":true},"execution_count":null,"outputs":[]}]}