{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# !pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl'\n!pip install --no-deps \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install --no-deps \"../input/resnest/resnest-0.0.5-py3-none-any.whl\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torch.utils.data.sampler import SequentialSampler\n\n# import ensemble_boxes\nfrom  torchvision.models.utils import load_state_dict_from_url\nfrom torchvision.models.detection.backbone_utils import resnet_fpn_backbone\nfrom torchvision.models.detection.backbone_utils import BackboneWithFPN\nfrom torchvision.ops import misc as misc_nn_ops\nfrom collections import OrderedDict\nfrom torch.jit.annotations import Tuple, List, Dict, Optional\n\nfrom resnest.torch import resnest101","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timmeffdetpkg\")\nsys.path.insert(0, \"../input/omegaconf\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\nsys.path.append(\"../input/wheatdetectiontta/external/pytorch-image-models\")\nsys.path.append(\"../input/wheatdetectiontta\")\nsys.path.append(\"../input/wheatdetectiontta/external/yacs\")\nimport torch\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom glob import glob\n\n\nfrom ensemble_boxes import *\nimport gc\nfrom tqdm import tqdm\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval, DetBenchTrain\nfrom effdet.efficientdet import HeadNet\nfrom skopt import gp_minimize, forest_minimize\nfrom skopt.utils import use_named_args\nfrom skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret\nfrom skopt.space import Categorical, Integer, Real\n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        \n        if self.test or random.random() > 0.5:\n            image, boxes = self.load_image_and_boxes(index)\n        else:\n            image, boxes = self.load_cutmix_image_and_boxes(index)\n\n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return image, boxes\n\n    def load_cutmix_image_and_boxes(self, index, imsize=1024):\n        \"\"\" \n        This implementation of cutmix author:  https://www.kaggle.com/nvnnghia \n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize\n        s = imsize // 2\n    \n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y\n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]\n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32)\n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)\n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n\n            boxes[:, 0] += padw\n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n\n        result_boxes = np.concatenate(result_boxes, 0)\n        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[np.where((result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1]) > 0)]\n        return result_image, result_boxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class TestDatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, path, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n        self.path = path\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{self.path}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        #image /= 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def get_df_folds(marking):\n\n    df_folds = marking[['image_id']].copy()\n    df_folds.loc[:, 'bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds.loc[:, 'stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n    df_folds.loc[:, 'fold'] = 0\n\n#     for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n#         df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n    \n    return df_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\n\ndef get_train_transforms(size):\n    return A.Compose(\n        [\n            A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),\n            A.OneOf([\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, \n                                     val_shift_limit=0.2, p=0.9),\n                A.RandomBrightnessContrast(brightness_limit=0.2, \n                                           contrast_limit=0.2, p=0.9),\n            ],p=0.9),\n            A.ToGray(p=0.01),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=size, width=size, p=1),\n            A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n\ndef get_valid_transforms(size):\n    return A.Compose(\n        [\n            A.Resize(height=size, width=size, p=1.0),\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resnest","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class CrossEntropyLabelSmooth(torch.nn.Module):\n    \"\"\"Cross entropy loss with label smoothing regularizer.\n\n    Reference:\n    Szegedy et al. Rethinking the Inception Architecture for Computer Vision. CVPR 2016.\n    Equation: y = (1 - epsilon) * y + epsilon / K.\n\n    Args:\n        num_classes (int): number of classes.\n        epsilon (float): weight.\n    \"\"\"\n    def __init__(self, num_classes, epsilon=0.1, use_gpu=True):\n        super(CrossEntropyLabelSmooth, self).__init__()\n        self.num_classes = num_classes\n        self.epsilon = epsilon\n        self.use_gpu = use_gpu\n        self.logsoftmax = nn.LogSoftmax(dim=1)\n\n    def forward(self, inputs, targets):\n        \"\"\"\n        Args:\n            inputs: prediction matrix (before softmax) with shape (batch_size, num_classes)\n            targets: ground truth labels with shape (num_classes)\n        \"\"\"\n        log_probs = self.logsoftmax(inputs)\n        targets = torch.zeros(log_probs.size()).scatter_(1, targets.unsqueeze(1).data.cpu(), 1)\n        if self.use_gpu: targets = targets.cuda()\n        targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes\n        loss = (- targets * log_probs).mean(0).sum()\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fastrcnn_loss(class_logits, box_regression, labels, regression_targets):\n    # type: (Tensor, Tensor, List[Tensor], List[Tensor]) -> Tuple[Tensor, Tensor]\n    \"\"\"\n    Computes the loss for Faster R-CNN.\n    Arguments:\n        class_logits (Tensor)\n        box_regression (Tensor)\n        labels (list[BoxList])\n        regression_targets (Tensor)\n    Returns:\n        classification_loss (Tensor)\n        box_loss (Tensor)\n    \"\"\"\n\n    labels = torch.cat(labels, dim=0)\n    regression_targets = torch.cat(regression_targets, dim=0)\n    labal_smooth_loss = CrossEntropyLabelSmooth(2)\n    classification_loss = labal_smooth_loss(class_logits, labels)\n\n    # get indices that correspond to the regression targets for\n    # the corresponding ground truth labels, to be used with\n    # advanced indexing\n    sampled_pos_inds_subset = torch.nonzero(labels > 0).squeeze(1)\n    labels_pos = labels[sampled_pos_inds_subset]\n    N, num_classes = class_logits.shape\n    box_regression = box_regression.reshape(N, -1, 4)\n\n    box_loss = det_utils.smooth_l1_loss(\n        box_regression[sampled_pos_inds_subset, labels_pos],\n        regression_targets[sampled_pos_inds_subset],\n        beta=1 / 9,\n        size_average=False,\n    )\n    box_loss = box_loss / labels.numel()\n\n    return classification_loss, box_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resnest_fpn_backbone(pretrained, norm_layer=misc_nn_ops.FrozenBatchNorm2d, trainable_layers=3):\n    # resnet_backbone = resnet.__dict__['resnet152'](pretrained=pretrained,norm_layer=norm_layer)\n    backbone = resnest101(pretrained=pretrained)\n    # select layers that wont be frozen\n    assert trainable_layers <= 5 and trainable_layers >= 0\n    layers_to_train = ['layer4', 'layer3', 'layer2', 'layer1', 'conv1'][:trainable_layers]\n    # freeze layers only if pretrained backbone is used\n    for name, parameter in backbone.named_parameters():\n        if all([not name.startswith(layer) for layer in layers_to_train]):\n            parameter.requires_grad_(False)\n    return_layers = {'layer1': '0', 'layer2': '1', 'layer3': '2', 'layer4': '3'}\n    in_channels_stage2 = backbone.inplanes // 8\n    in_channels_list = [\n        in_channels_stage2,\n        in_channels_stage2 * 2,\n        in_channels_stage2 * 4,\n        in_channels_stage2 * 8,\n    ]\n    out_channels = 256\n    return BackboneWithFPN(backbone, return_layers, in_channels_list, out_channels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WheatDetector(torch.nn.Module):\n    def __init__(self, trainable_layers=3, **kwargs):\n        super(WheatDetector, self).__init__()\n        backbone = resnest_fpn_backbone(pretrained=False)\n        self.base = FasterRCNN(backbone, num_classes = 2, **kwargs)\n        self.base.roi_heads.fastrcnn_loss = fastrcnn_loss\n\n    def forward(self, images, targets=None):\n        return self.base(images, targets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from modeling import build_model\nfrom config import cfg\n\ndef load_res_net(path, cfg):\n    model = build_model(cfg)\n    checkpoint = torch.load(path)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    model.eval();\n    return model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_net(level):\n    config = get_efficientdet_config(f'tf_efficientdet_d{level}')\n    net = EfficientDet(config, pretrained_backbone=False)\n    \n    if level == 5:\n        checkpoint = torch.load('../input/efficientdet/efficientdet_d5-ef44aea8.pth')\n    elif level == 7:\n        checkpoint = torch.load('../input/efficientdet/efficientdet_d7-f05bf714.pth')\n        \n    net.load_state_dict(checkpoint)\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n    net = DetBenchTrain(net, config)\n    return net\n\ndef load_net_eval(checkpoint_path, level):\n    config = get_efficientdet_config(f'tf_efficientdet_d{level}')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint[\"model_state_dict\"])\n\n    del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\n#net = get_net()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fitter","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:            \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nclass Fitter:\n    \n    def __init__(self, model, device, config):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{config.folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n        \n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n\n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = RAdam(self.model.parameters(), lr=config.lr)\n        #self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch(train_loader)\n\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/retrain.bin')\n\n            t = time.time()\n            summary_loss = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-retrain.bin')\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                images = torch.stack(images)\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                boxes = [target['boxes'].to(self.device).float() for target in targets]\n                labels = [target['labels'].to(self.device).float() for target in targets]\n\n                loss, _, _ = self.model(images, boxes, labels)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        return summary_loss\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            images = torch.stack(images)\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n            boxes = [target['boxes'].to(self.device).float() for target in targets]\n            labels = [target['labels'].to(self.device).float() for target in targets]\n\n            self.optimizer.zero_grad()\n            \n            loss, _, _ = self.model(images, boxes, labels)\n            \n            loss.backward()\n\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss\n    \n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.model.state_dict(),\n            #'optimizer_state_dict': self.optimizer.state_dict(),\n            #'scheduler_state_dict': self.scheduler.state_dict(),\n            #'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path, device):\n        checkpoint = torch.load(path)\n        self.model.model.load_state_dict(checkpoint['model_state_dict'])\n        self.model.to(device)\n        #self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        #self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        #self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Global Setting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"N_FOLD = 2\nUSE_TTA = True\nTRAIN_ROOT_PATH = '../input/global-wheat-detection/train'\nTEST_ROOT_PATH = '../input/global-wheat-detection/test'\ncfg.MODEL.PRETRAIN = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = {\n    \"effdet\": [\n        load_net_eval('../input/effdetd5sourcelee/best-retrain-epoch51.bin', 5), \n        load_net_eval('../input/effdetd7/retrains/best-retrain-epoch42.bin', 7)\n    ],\n    \"resnest\":[\n        load_res_net(\"../input/resnest-source-weights-andreshuang/checkpoint-60arvalis1.bin\", cfg),\n        load_res_net(\"../input/rssnest-source-weights-cjc/best-checkpoint_PL_ethz_1.bin\", cfg)\n    ]\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bayesian optimization using Gaussian Processes","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numba\nimport re\nimport ast\n\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision\n\ndef show_result(sample_id, preds, gt_boxes):\n    sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)\n    sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for pred_box in preds:\n        cv2.rectangle(\n            sample,\n            (pred_box[0], pred_box[1]),\n            (pred_box[2], pred_box[3]),\n            (220, 0, 0), 2\n        )\n\n    for gt_box in gt_boxes:    \n        cv2.rectangle(\n            sample,\n            (gt_box[0], gt_box[1]),\n            (gt_box[2], gt_box[3]),\n            (0, 0, 220), 2\n        )\n\n    ax.set_axis_off()\n    ax.imshow(sample)\n    ax.set_title(\"RED: Predicted | BLUE - Ground-truth\")\n    \n# Numba typed list!\niou_thresholds = numba.typed.List()\n\nfor x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n    iou_thresholds.append(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def to_tensor(images):\n    tmp = []\n    for img in images:\n        img = img.astype(np.float32)\n        img /= 255.0\n        img = torch.tensor(img, dtype=torch.float32)\n        tmp.append(img.permute(2,0,1))\n    return torch.stack(tmp)\n\ndef get_handout_transforms():\n    return A.Compose(\n        [\n            A.Resize(height=512, width=512, p=1.0),\n            #ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n\nclass HandoutDatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, path):\n        super().__init__()\n        self.image_ids = image_ids\n        self.marking = marking\n        self.path = path\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{self.path}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        records = self.marking[self.marking['image_id'] == image_id]\n        gtboxes = records[['x', 'y', 'w', 'h']].values\n        gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]\n        gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]\n            \n        return image, gtboxes, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_predictions(\n    models,\n    images, \n    score_threshold=0.25,\n):\n    predictions = []\n    for fold_number, net in enumerate(models):\n        with torch.no_grad():\n            net.eval()\n            det = net(images, torch.tensor([1]*images.shape[0]).float().cuda())\n            result = []\n            for i in range(images.shape[0]):\n                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                scores = det[i].detach().cpu().numpy()[:,4]\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                result.append({\n                    'boxes': boxes[indexes],\n                    'scores': scores[indexes],\n                })\n            predictions.append(result)\n    return predictions\n\n\ndef run_wbf(predictions, image_index, image_size=512, iou_thr=0.5, skip_box_thr=0.4, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*1023\n    return boxes, scores, labels\n\n\ndef TTAImage(image, index):\n    image1 = image.copy()\n    if index==0: \n        rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image\n    elif index==1:\n        rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image2\n    elif index==2:\n        rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image3\n    elif index == 3:\n        return image1\n    \n    \ndef rotBoxes90(boxes, im_w, im_h):\n    ret_boxes =[]\n    for box in boxes:\n        x1, y1, x2, y2 = box\n        x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2\n        x1, y1, x2, y2 = y1, -x1, y2, -x2\n        x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)\n        x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)\n        ret_boxes.append([x1a, y1a, x2a, y2a])\n    return np.array(ret_boxes)\n\n\ndef run_wbf_tta(predictions, image_size=512, iou_thr=0.5, skip_box_thr=0.4, weights=None):\n    boxes_1 = []\n    scores_1 = []\n    labels_1 = []\n    for i in range(2):\n        bb = (predictions['boxes'][i]/(image_size-1)).tolist()\n        ss = predictions['scores'][i].tolist()\n        ls = predictions['labels'][i].tolist()\n        boxes_1.append(bb)\n        scores_1.append(ss)\n        labels_1.append(ls)\n\n    boxes_1, scores_1, labels_1 = weighted_boxes_fusion(boxes_1, scores_1, labels_1, \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    boxes_2 = []\n    scores_2 = []\n    labels_2 = []\n    for i in range(2,4):\n        bb = (predictions['boxes'][i]/(image_size-1)).tolist()\n        ss = predictions['scores'][i].tolist()\n        ls = predictions['labels'][i].tolist()\n        boxes_2.append(bb)\n        scores_2.append(ss)\n        labels_2.append(ls)\n    \n    boxes_2, scores_2, labels_2 = weighted_boxes_fusion(boxes_2, scores_2, labels_2, \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    boxes = [boxes_1.tolist(), boxes_2.tolist()]\n    scores = [scores_1.tolist(), scores_2.tolist()]\n    labels = [labels_1.tolist(), labels_2.tolist()]\n    \n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    #boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\n\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_final_score(\n    all_predictions,\n    iou_thr,\n    skip_box_thr,\n    resweight,\n    edetweight,\n    sigma=0.5,\n):\n    final_scores = []\n    for predictions in all_predictions:\n        \n        gt_boxes = predictions['gtboxes'].copy()\n        image_id = predictions['image_id']\n        \n        img_boxes = []\n        img_scores = []\n        img_labels = []\n        \n        for index in range(4):\n            effdet_p = predictions[\"effdet\"][index]\n            boxes, scores, labels = run_wbf(effdet_p, image_size=512, image_index=0, \n                                            iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n            \n            resnest_p = predictions[\"resnest\"][index]\n            boxes = [resnest_p[\"boxes\"], boxes]\n            scores = [resnest_p[\"scores\"], scores]\n            labels = [resnest_p[\"labels\"], labels]\n            \n            boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, \n                                            weights=[resweight, edetweight], iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n            \n            for _ in range(3-index):\n\n                boxes = rotBoxes90(boxes, 1024, 1024)\n\n            if index == 3:\n\n                boxes = boxes.astype(np.int32)\n\n            if index == 0:\n\n                img_boxes = (boxes / 1024).tolist()\n                img_scores = scores.tolist()\n                img_labels = labels.tolist()\n\n            else:\n\n                img_boxes = img_boxes + (boxes / 1024).tolist()\n                img_scores = img_scores + scores.tolist()\n                img_labels = img_labels + labels.tolist()\n\n        boxes, scores, labels = weighted_boxes_fusion([img_boxes], \n                                                      [img_scores], \n                                                      [img_labels], \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n        boxes = (boxes*1024).astype(np.int32).clip(min=0, max=1023)\n        image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')\n        \n        final_scores.append(image_precision)\n    \n    score = np.mean(final_scores)\n    \n    print(f\"score:{score}  iou_thr:{iou_thr}  skip_box_thr:{skip_box_thr}  weights:[{resweight, edetweight}]\")\n    \n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"USE_OPTIMIZE = False\n\nif USE_OPTIMIZE:\n    \n    marking = pd.read_csv('../input/global-wheat-detection/train.csv')\n\n    bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        marking[column] = bboxs[:,i]\n    marking.drop(columns=['bbox'], inplace=True)\n\n    # drop some source\n    # drop_source = [\"ethz_1\", \"arvalis_1\"]\n    # marking = marking[~marking['source'].isin(drop_source)]\n\n    # drop error boxes\n    marking['area'] = marking['w'] * marking['h']\n    marking = marking[marking['area'] < 154200.0]\n    error_boxes = [100648.0, 145360.0, 149744.0,119790.0, 106743.0]\n    marking = marking[~marking['area'].isin(error_boxes)]\n\n    # drop less boxes images\n    all_train_images = marking.copy()\n    all_train_images['count'] = all_train_images.apply(lambda row: 1 if np.isfinite(row.width) else 0, axis=1)\n    train_images_count = all_train_images.groupby('image_id').sum().reset_index()\n    indexes = train_images_count[train_images_count['count'] < 10].image_id\n\n    marking = marking[~marking['image_id'].isin(indexes)]\n    \n    skf = StratifiedKFold(n_splits=4, shuffle=True, random_state=42)\n\n    df_folds = marking[['image_id']]\n    df_folds['bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds['source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds['stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n\n    train_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))\n    df_holdout = df_folds.iloc[holdout_index].copy()\n    df_folds = df_folds.iloc[train_index].copy()\n\n    df_folds['fold'] = 0\n    for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n        df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n    \n    holdout_dataset = HandoutDatasetRetriever(\n        image_ids=df_holdout.index.values,\n        marking=marking,\n        path=TRAIN_ROOT_PATH,\n    )\n    \n    holdout_loader = DataLoader(\n        holdout_dataset,\n        batch_size=1,\n        shuffle=False,\n        num_workers=4,\n        drop_last=False,\n        collate_fn=collate_fn\n    )\n    \n    all_predictions = []\n    for images, gtboxes, image_ids in tqdm(holdout_loader, total=len(holdout_loader)):\n\n        image = images[0]\n        image = cv2.resize(image, (512, 512))\n        image_res = images[0]\n\n        predictions_tta = {\"effdet\":[],\n                          \"resnest\":[],\n                          \"gtboxes\":gtboxes[0],\n                          \"image_id\":image_ids[0]}\n\n        for index in range(4):\n            \n            roated = TTAImage(image, index)\n            roated = to_tensor([roated]).cuda()\n            predictions = make_predictions(MODEL[\"effdet\"], roated)\n            \n            del roated\n            \n            predictions_tta[\"effdet\"].append(predictions)\n\n            roated = TTAImage(image_res, index)\n            roated = to_tensor([roated]).cuda()\n\n            predictions = {}\n            predictions_1 = MODEL[\"resnest\"][0](roated)\n            predictions_2 = MODEL[\"resnest\"][1](roated)\n            predictions[\"boxes\"] = predictions_1[0][\"boxes\"].tolist() + predictions_2[0][\"boxes\"].tolist()\n            predictions[\"scores\"] = predictions_1[0][\"scores\"].tolist() + predictions_2[0][\"scores\"].tolist()\n            predictions[\"labels\"] = predictions_1[0][\"labels\"].tolist() + predictions_2[0][\"labels\"].tolist()\n            predictions_tta[\"resnest\"].append(predictions)\n            del roated\n            \n        all_predictions.append(predictions_tta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def log(text):\n    with open('opt.log', 'a+') as logger:\n        logger.write(f'{text}\\n')\n        \n        \ndef optimize(space, all_predictions, n_calls=10):\n    @use_named_args(space)\n    def score(**params):\n        log('-'*5 + 'WBF' + '-'*5)\n        log(params)\n        final_score = calculate_final_score(all_predictions, **params)\n        log(f'final_score = {final_score}')\n        log('-'*10)\n        return -final_score\n\n    return gp_minimize(func=score, dimensions=space, n_calls=n_calls)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"if USE_OPTIMIZE:\n    space = [\n        Real(0.1, 0.7, name='iou_thr'),\n        Real(0.2, 0.7, name='skip_box_thr'),\n        Real(1, 10, name='resweight'),\n        Real(1, 10, name='edetweight'),\n    ]\n\n    opt_result = optimize(\n        space, \n        all_predictions,\n        n_calls=10,\n    )\n    \n    best_final_score = -opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\n    \n    \nelse:\n    # calculated early for fast inference in submission [see version 1 with n_calls=300]\n    best_final_score = 0.68687\n    best_iou_thr = 0.41485\n    best_skip_box_thr = 0.4159\n    best_resweight = 3.62106\n    best_edetweight = 6.50667\n    \nprint('-'*13 + 'WBF' + '-'*14)\nprint(f'[Best Score]: {best_final_score:.4f}')\nprint(f'[Best Iou Thr]: {best_iou_thr:.3f}')\nprint(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\nprint(f'[Best Best Resnest weight]: {best_resweight:.3f}')\nprint(f'[Best Best Effdet weight]: {best_edetweight:.3f}')\n\nprint('-'*30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Pseudo Label","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def make_predictions(\n    models,\n    images, \n    score_threshold=0.25,\n):\n    predictions = []\n    for fold_number, net in enumerate(models):\n        with torch.no_grad():\n            net.eval()\n            det = net(images, torch.tensor([1]*images.shape[0]).float().cuda())\n            result = []\n            for i in range(images.shape[0]):\n                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                scores = det[i].detach().cpu().numpy()[:,4]\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                result.append({\n                    'boxes': boxes[indexes],\n                    'scores': scores[indexes],\n                })\n            predictions.append(result)\n    return predictions\n\n\ndef run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*1023\n    return boxes, scores, labels\n\n\ndef TTAImage(image, index):\n    image1 = image.copy()\n    if index==0: \n        rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image\n    elif index==1:\n        rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image2\n    elif index==2:\n        rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image3\n    elif index == 3:\n        return image1\n    \n    \ndef rotBoxes90(boxes, im_w, im_h):\n    ret_boxes =[]\n    for box in boxes:\n        x1, y1, x2, y2 = box\n        x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2\n        x1, y1, x2, y2 = y1, -x1, y2, -x2\n        x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)\n        x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)\n        ret_boxes.append([x1a, y1a, x2a, y2a])\n    return np.array(ret_boxes)\n\n\ndef run_wbf_tta(predictions, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):\n    boxes_1 = []\n    scores_1 = []\n    labels_1 = []\n    for i in range(2):\n        bb = (predictions['boxes'][i]/(image_size-1)).tolist()\n        ss = predictions['scores'][i].tolist()\n        ls = predictions['labels'][i].tolist()\n        boxes_1.append(bb)\n        scores_1.append(ss)\n        labels_1.append(ls)\n\n    boxes_1, scores_1, labels_1 = weighted_boxes_fusion(boxes_1, scores_1, labels_1, \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    boxes_2 = []\n    scores_2 = []\n    labels_2 = []\n    for i in range(2,4):\n        bb = (predictions['boxes'][i]/(image_size-1)).tolist()\n        ss = predictions['scores'][i].tolist()\n        ls = predictions['labels'][i].tolist()\n        boxes_2.append(bb)\n        scores_2.append(ss)\n        labels_2.append(ls)\n    \n    boxes_2, scores_2, labels_2 = weighted_boxes_fusion(boxes_2, scores_2, labels_2, \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    boxes = [boxes_1.tolist(), boxes_2.tolist()]\n    scores = [scores_1.tolist(), scores_2.tolist()]\n    labels = [labels_1.tolist(), labels_2.tolist()]\n    \n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, \n                                                        weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    #boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\n\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = TestDatasetRetriever(\n    image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_ROOT_PATH}/*.jpg')]),\n    path=TEST_ROOT_PATH\n)\n\ntest_data_loader = DataLoader(\n    test_dataset,\n    batch_size=1,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if len(os.listdir(TEST_ROOT_PATH)) > 10:\n    \n    marking = pd.read_csv('../input/global-wheat-detection/train.csv')\n\n    bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        marking[column] = bboxs[:,i]\n    marking.drop(columns=['bbox'], inplace=True)\n\n    marking_p = marking.copy()\n    results = []\n    for images, image_ids in test_data_loader:\n\n        image = images[0]\n        height, width, _ = image.shape\n        image = cv2.resize(image, (512, 512))\n        image_res = cv2.resize(image, (1024, 1024))\n\n        predictions_tta = {\n            \"boxes\": [],\n            \"scores\": [],\n            \"labels\": []\n        }\n\n        for index in range(4):\n            roated = TTAImage(image, index)\n            roated = to_tensor([roated]).cuda()\n            predictions = make_predictions(MODEL[\"effdet\"], roated)\n            boxes, scores, labels = run_wbf(predictions, image_size=512, image_index=0)\n\n\n            roated = TTAImage(image_res, index)\n            roated = to_tensor([roated]).cuda()\n            predictions_1 = MODEL[\"resnest\"][0](roated)\n            predictions_2 = MODEL[\"resnest\"][1](roated)\n            boxes_r = predictions_1[0][\"boxes\"].tolist() + predictions_2[0][\"boxes\"].tolist()\n            scores_r = predictions_1[0][\"scores\"].tolist() + predictions_2[0][\"scores\"].tolist()\n            labels_r = predictions_1[0][\"labels\"].tolist() + predictions_2[0][\"labels\"].tolist()\n\n            boxes = [boxes_r, boxes]\n            scores = [scores_r, scores]\n            labels = [labels_r, labels]\n\n            boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, \n                                                          weights=[best_resweight, best_edetweight], \n                                                          iou_thr=best_iou_thr, \n                                                          skip_box_thr=best_skip_box_thr)\n\n            for _ in range(3-index):\n\n                boxes = rotBoxes90(boxes, 1024, 1024)\n\n            if index == 3:\n\n                boxes = boxes.astype(np.int32)\n\n            predictions_tta[\"boxes\"].append(boxes)\n            predictions_tta[\"scores\"].append(scores)\n            predictions_tta[\"labels\"].append(labels)\n\n        boxes, scores, labels = run_wbf_tta(predictions_tta, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)\n\n        boxes[:, [0, 1]] = boxes[:, [0, 1]] - boxes[:, [0, 1]]*0.01\n        boxes[:, [2, 3]] = boxes[:, [2, 3]] + boxes[:, [2, 3]]*0.01\n\n        boxes = (boxes*1024).astype(np.int32).clip(min=0, max=1023)\n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n\n        for bb, ss in zip(boxes, scores):\n            marking_p = marking_p.append({\n             'image_id':image_ids[0], \n                'width':1024, \n                'height':1024, \n                'source':\"arvalis_1\", \n                'x':bb[0], 'y':bb[1], 'w':bb[2], 'h':bb[3]\n            }, ignore_index=True)\n\n    marking_p = marking_p[~marking_p['image_id'].isin(marking.image_id)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Retrain model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"N_FOLD = 2\nUSE_OPTIMIZE = False # used for fast inference in submission\nUSE_TTA = True\nTRAIN_ROOT_PATH = '../input/global-wheat-detection/test'\nTEST_ROOT_PATH = '../input/global-wheat-detection/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers = 8\n    batch_size = 4\n    n_epochs = 5 # n_epochs = 40\n    lr = 0.0002\n\n    folder = 'retrains'\n\n    # -------------------\n    verbose = True\n    verbose_step = 1\n    # -------------------\n\n    # --------------------\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n\n#     SchedulerClass = torch.optim.lr_scheduler.OneCycleLR\n#     scheduler_params = dict(\n#         max_lr=0.001,\n#         epochs=n_epochs,\n#         steps_per_epoch=int(len(train_dataset) / batch_size),\n#         pct_start=0.1,\n#         anneal_strategy='cos', \n#         final_div_factor=10**5\n#     )\n    \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=1,\n        verbose=False, \n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-8,\n        eps=1e-08\n    )\n    # --------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def re_train(path, marker, level, folder=None):\n    \n    df_folds = get_df_folds(marker)\n    device = torch.device('cuda:0')\n    net = get_net(level)\n    \n    if folder:\n        TrainGlobalConfig.folder = folder\n    \n    train_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] == 0].index.values,\n        marking=marker,\n        transforms=get_train_transforms(512),\n        test=True,\n    )\n\n    validation_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] == 0].index.values[:len(df_folds) // 5],\n        marking=marker,\n        transforms=get_valid_transforms(512),\n        test=True,\n    )\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        sampler=RandomSampler(train_dataset),\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n        collate_fn=collate_fn,\n    )\n    \n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_fn,\n    )\n\n    \n    fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)\n    \n    fitter.load(path, device)\n    \n    fitter.fit(train_loader, val_loader)\n    \n    del fitter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if len(os.listdir(TEST_ROOT_PATH)) > 10:\n    \n    re_train(\"../input/effdetd5sourcelee/best-retrain-epoch51.bin\", marking_p, 5, \"retrains\")\n    \n    MODEL[\"effdet\"][0] = load_net_eval(\"retrains/best-retrain.bin\", 5)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\nfor images, image_ids in test_data_loader:\n\n    image = images[0]\n    height, width, _ = image.shape\n    image = cv2.resize(image, (512, 512))\n    image_res = cv2.resize(image, (1024, 1024))\n\n    predictions_tta = {\n        \"boxes\": [],\n        \"scores\": [],\n        \"labels\": []\n    }\n\n    for index in range(4):\n        roated = TTAImage(image, index)\n        roated = to_tensor([roated]).cuda()\n        predictions = make_predictions(MODEL[\"effdet\"], roated)\n        boxes, scores, labels = run_wbf(predictions, image_size=512, image_index=0)\n        \n        \n        roated = TTAImage(image_res, index)\n        roated = to_tensor([roated]).cuda()\n        predictions_1 = MODEL[\"resnest\"][0](roated)\n        predictions_2 = MODEL[\"resnest\"][1](roated)\n        boxes_r = predictions_1[0][\"boxes\"].tolist() + predictions_2[0][\"boxes\"].tolist()\n        scores_r = predictions_1[0][\"scores\"].tolist() + predictions_2[0][\"scores\"].tolist()\n        labels_r = predictions_1[0][\"labels\"].tolist() + predictions_2[0][\"labels\"].tolist()\n        \n        boxes = [boxes_r, boxes]\n        scores = [scores_r, scores]\n        labels = [labels_r, labels]\n        \n        boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, \n                                                      weights=[best_resweight, best_edetweight], \n                                                      iou_thr=best_iou_thr, \n                                                      skip_box_thr=best_skip_box_thr)\n        \n        for _ in range(3-index):\n\n            boxes = rotBoxes90(boxes, 1024, 1024)\n\n        if index == 3:\n\n            boxes = boxes.astype(np.int32)\n\n        predictions_tta[\"boxes\"].append(boxes)\n        predictions_tta[\"scores\"].append(scores)\n        predictions_tta[\"labels\"].append(labels)\n\n    boxes, scores, labels = run_wbf_tta(predictions_tta, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)\n    \n    boxes[:, [0, 1]] = boxes[:, [0, 1]] - boxes[:, [0, 1]]*0.01\n    boxes[:, [2, 3]] = boxes[:, [2, 3]] + boxes[:, [2, 3]]*0.01\n    \n    scores = scores.clip(min=0, max=1.0)\n    boxes = (boxes*1024).astype(np.int32).clip(min=0, max=1023)\n    boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n            \n#     img_ = cv2.resize(images[0], (1024, 1024))\n#     for bb, ss in zip(boxes, scores):\n#         cv2.rectangle(img_, (bb[0], bb[1]), (bb[2]+bb[0], bb[3]+bb[1]), (220, 0, 0), 2)\n#         cv2.putText(img_, '%.2f'%(ss), (bb[0], bb[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,255,255), 2, cv2.LINE_AA)\n    \n#     plt.figure(figsize=(16, 8))\n#     plt.imshow(img_)\n#     plt.axis(\"off\")\n    \n    result = {\n            'image_id': image_ids[0],\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n    \n    results.append(result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)\ntest_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}