{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\nimport os\nimport ensemble_boxes\nimport torch\nimport random\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom torch.utils.data import Dataset,DataLoader\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport cv2\nimport gc\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval, DetBenchTrain\nfrom effdet.efficientdet import HeadNet\nfrom sklearn.model_selection import StratifiedKFold\nfrom skopt import gp_minimize, forest_minimize\nfrom skopt.utils import use_named_args\nfrom skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret\nfrom skopt.space import Categorical, Integer, Real\nfrom sklearn.model_selection import StratifiedKFold\n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_score_threshold = 0.39\nbest_iou_thr=0.41\nbest_skip_box_thr=0.41\nSCORE_OPTIM = len(os.listdir('../input/global-wheat-detection/test/'))>11\nWBF_OPTIM = False#len(os.listdir('../input/global-wheat-detection/test/'))>11","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1. Prepare Test Dataloader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_DATA_ROOT_PATH = '../input/global-wheat-detection/test'\n# TEST_DATA_ROOT_PATH = '../input/gwd-test/'\n\nTRAIN_DATA_ROOT_PATH = '../input/global-wheat-detection/train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_transforms():\n    return A.Compose([\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TEST_DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = TestDatasetRetriever(\n    image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_DATA_ROOT_PATH}/*.jpg')]),\n    transforms=get_test_transforms()\n)\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ntest_data_loader = DataLoader(\n    test_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=2,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Prepare Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_net(checkpoint_path):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size=512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint['model_state_dict'])\n\n    del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. TTA(Test Time Augmentation)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"> 为原始图像造出多个不同版本，包括不同区域裁剪和更改缩放程度等，并将它们输入到模型中；然后对多个版本进行计算得到平均输出，作为图像的最终输出分数。","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = 512\n    \n    def augment(self,image):\n        raise NotImplementedError\n        \n    def batch_augment(self,images):\n        raise NotImplementedError\n        \n    def deaugment_boxes(self,boxes):\n        raise NotImplementedError\n        \nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self,image):\n        return image.flip(1)\n    \n    def batch_augment(self,images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self,boxes):\n        boxes[:,[1,3]] = self.image_size - boxes[:,[3,1]]\n        return boxes\n    \nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n    \nclass TTARotate180(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 2, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 2, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]\n        return boxes\n    \nclass TTARotate270(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 3, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 3, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]\n        return res_boxes\n    \nclass TTAHSV_or_RBC(BaseWheatTTA):\n    \"\"\"Random change HSV or BC\"\"\"\n    transform = A.OneOf([\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, \n                                     val_shift_limit=0.2,p=0.5),\n                A.RandomBrightnessContrast(brightness_limit=0.2, \n                                           contrast_limit=0.2, p=0.5),\n            ],p=1)\n    \n    def augment(self,image):\n        image = image.permute(1,2,0).cpu().numpy()\n        image = self.transform(image=image)['image']\n        return ToTensorV2()(image=image)['image']\n\n    def batch_augment(self, images):\n        batch_size = len(images)\n        for i in range(batch_size):\n            images[i] = self.augment(images[i])\n        return images\n        \n    def deaugment_boxes(self,boxes):\n        return boxes\n\nclass TTAToGray(BaseWheatTTA):\n    \"\"\"To Gray\"\"\"\n    \n    transform = A.ToGray()\n    def augment(self,image):\n        image = image.permute(1,2,0).cpu().numpy()\n        image = self.transform(image=image)['image']\n        return ToTensorV2()(image=image)['image']\n\n    def batch_augment(self, images):\n        batch_size = len(images)\n        for i in range(batch_size):\n            images[i] = self.augment(images[i])\n        return images\n        \n    def deaugment_boxes(self,boxes):\n        return boxes\n    \nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_det(index, det, score_threshold=0.25):\n    boxes = det[index].detach().cpu().numpy()[:,:4]    \n    scores = det[index].detach().cpu().numpy()[:,4]\n    boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n    boxes = (boxes).clip(min=0, max=511).astype(int)\n    indexes = np.where(scores>score_threshold)\n    boxes = boxes[indexes]\n    scores = scores[indexes]\n    return boxes, scores","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. Combinations of TTA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import product\n\ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(),None],\n                               [TTAVerticalFlip(),None],\n                               [TTARotate90(),None],\n#                                [TTAHSV_or_RBC(),None]\n                               ):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))\n    \n# for tta_combination in product([None], \n#                                [TTARotate90(), TTARotate180(), TTARotate270(), None]):\n#     tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. Other functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_tta_predictions(images, score_threshold=best_score_threshold,nets=None):\n#     print(score_threshold)\n    with torch.no_grad():\n        images = torch.stack(images).float().cuda()\n        predictions = []\n        for net in nets:\n            for tta_transform in tta_transforms:\n                result = []\n                det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0]).float().cuda())\n\n                for i in range(images.shape[0]):\n                    boxes = det[i].detach().cpu().numpy()[:,:4]    \n                    scores = det[i].detach().cpu().numpy()[:,4]\n                    indexes = np.where(scores > score_threshold)[0]\n                    boxes = boxes[indexes]\n                    boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                    boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                    boxes = tta_transform.deaugment_boxes(boxes.copy())\n                    result.append({\n                        'boxes': boxes,\n                        'scores': scores[indexes],\n                    })\n                predictions.append(result)\n    return predictions\n\ndef run_wbf(predictions, image_index,image_size=512,iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr,   weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in predictions]\n    boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### these↓ functions are used for evaluation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numba\nimport re\nimport ast\nimport matplotlib.pyplot as plt\n\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision\n\ndef show_result(sample_id, preds, gt_boxes):\n    sample = cv2.imread(f'{TRAIN_DATA_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)\n    sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for pred_box in preds:\n        cv2.rectangle(\n            sample,\n            (pred_box[0], pred_box[1]),\n            (pred_box[2], pred_box[3]),\n            (220, 0, 0), 2\n        )\n\n    for gt_box in gt_boxes:    \n        cv2.rectangle(\n            sample,\n            (gt_box[0], gt_box[1]),\n            (gt_box[2], gt_box[3]),\n            (0, 0, 220), 2\n        )\n\n    ax.set_axis_off()\n    ax.imshow(sample)\n    ax.set_title(\"RED: Predicted | BLUE - Ground-truth\")\n    \n# Numba typed list!\niou_thresholds = numba.typed.List()\n\nfor x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n    iou_thresholds.append(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. Predict label for test set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef predict_test(data_loader,nets,detection_threshold = 0.5):\n    testdf_pseudo = []\n    for images, image_ids in data_loader:\n        predictions = make_tta_predictions(images,nets=nets)\n        for i, image in enumerate(images):\n            boxes, scores, labels = run_wbf(predictions, image_index=i)\n            boxes = (boxes*2).round().astype(np.int32).clip(min=0, max=1023)\n            \n            boxes = boxes[scores >= detection_threshold].astype(np.int32)\n            scores = scores[scores >= detection_threshold]\n            \n            image_id = image_ids[i]\n\n            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n            for box in boxes:\n                result = {\n                    'image_id': 'gzyou'+image_id,\n                    'width':1024,\n                    'height':1024,\n                    'source': 'gzyou',\n                    'x': box[0],\n                    'y': box[1],\n                    'w': box[2],\n                    'h': box[3] \n                }\n                testdf_pseudo.append(result)\n    print(len(testdf_pseudo))\n    test_df_pseudo = pd.DataFrame(testdf_pseudo, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])\n    return test_df_pseudo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(best_score_threshold,best_iou_thr,best_skip_box_thr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nets = [load_net('../input/efficientdet-pseudo-best-checkpoint-fold1/best-checkpoint-014epoch.bin')]\n# nets = [load_net('../input/efficientdet-best/best-checkpoint-54epoch.bin')]\ntest_df_pseudo = predict_test(test_data_loader,nets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_pseudo.head()#[test_df_pseudo['image_id']=='gzyou51b3e36ab']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 7. Prepare new train dataset and validation dataset ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose(\n        [\n            A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),\n            A.OneOf([\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2,\n                                     val_shift_limit=0.2, p=0.9), \n                A.RandomBrightnessContrast(brightness_limit=0.2,  \n                                           contrast_limit=0.2, p=0.9),\n            ], p=0.9),\n            A.ToGray(p=0.01),  \n            A.HorizontalFlip(p=0.5), \n            A.VerticalFlip(p=0.5),  \n            A.Resize(height=512, width=512, p=1),  \n            A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),  \n            ToTensorV2(p=1.0),\n        ],\n        p=1.0,\n        bbox_params=A.BboxParams(  \n            format='pascal_voc',\n            min_area=0,\n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n\n\ndef get_valid_transforms():\n    return A.Compose(\n        [\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ],\n        p=1.0,\n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0,\n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n\n        if self.test or random.random() > 0.5:#if self.test or 'gzyou' in image_id or random.random() > 0.5:\n            image, boxes = self.load_image_and_boxes(index)\n        else:\n#             image, boxes = self.load_mixup_image_and_boxes(index)\n            image, boxes = self.load_cutmix_image_and_boxes(index)\n\n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n\n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                assert len(sample['bboxes']) == len(sample['labels']), 'not equal!'\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    target['boxes'][:, [0, 1, 2, 3]] = target['boxes'][:, [1, 0, 3, 2]]  # yxyx: be warning\n                    changed = True\n                    break\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        records = self.marking[self.marking['image_id'] == image_id]\n        if 'gzyou' in image_id:\n            image_id = image_id[5:]\n            image = cv2.imread(f'{TEST_DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        else:\n            image = cv2.imread(f'{TRAIN_DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n            \n            \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        # PLEASE! PLEASE! PLEASE! Please be much CAREFUL here.\n        \n#         records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3] \n        return image, boxes\n    \n#     def load_mixup_image_and_boxes(self, index, imsize=1024):\n#         image, boxes = self.load_image_and_boxes(index)\n#         r_image,r_boxes = self.load_image_and_boxes(random.randint(0, self.image_ids.shape[0] - 1))\n#         mixup_image = (image + r_image) / 2\n#         mixup_boxes = np.concatenate([boxes,r_boxes], 0)\n#         return mixup_image,mixup_boxes\n\n    def load_cutmix_image_and_boxes(self, index, imsize=1024):\n        \"\"\"\n        This implementation of cutmix author:  https://www.kaggle.com/nvnnghia\n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize \n        s = imsize // 2\n\n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  \n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]  \n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32) \n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)  \n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc \n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  \n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n\n            boxes[:, 0] += padw  \n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n\n        result_boxes = np.concatenate(result_boxes, 0)\n        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[\n            np.where((result_boxes[:, 2] - result_boxes[:, 0]) * (result_boxes[:, 3] - result_boxes[:, 1]) > 0)]\n        return result_image, result_boxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_new_marking(marking,test_df_pseudo):\n    \n    bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        marking[column] = bboxs[:,i]\n    marking.drop(columns=['bbox'], inplace=True)\n    new_marking = pd.concat([marking, test_df_pseudo])\n    \n    return new_marking","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 7.1 Split method1: validation dataset only includes original images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_train_valid_dataset1(marking,new_marking,test_df_pseudo,fold_number = 0):\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    df_folds = marking[['image_id']].copy()\n    df_folds.loc[:, 'bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds.loc[:, 'stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n    df_folds.loc[:, 'fold'] = 0\n\n    for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n        df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n    \n    train_dataset = DatasetRetriever(\n    image_ids=np.append(df_folds[df_folds['fold'] != fold_number].index.values,test_df_pseudo['image_id'].unique()),\n    marking=new_marking,\n    transforms=get_train_transforms(),\n    test=False,\n    )\n\n    validation_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] == fold_number].index.values,\n        marking=new_marking,\n        transforms=get_valid_transforms(),\n        test=True,\n    )\n    \n    return train_dataset,validation_dataset\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 7.2 Split method2: validation dataset  includes pseudo labeling images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"(worse results...)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_valid_dataset2(new_marking):\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n    df_folds = new_marking[['image_id']].copy()\n    df_folds.loc[:, 'bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds.loc[:, 'source'] = new_marking[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds.loc[:, 'stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n    df_folds.loc[:, 'fold'] = 0\n\n    for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n        df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n\n\n    train_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] != fold_number].index.values,\n        marking=new_marking,\n        transforms=get_train_transforms(),\n        test=False,\n    )\n\n    validation_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] == fold_number].index.values,\n        marking=new_marking,\n        transforms=get_valid_transforms(),\n        test=True,\n    )\n    return train_dataset,validation_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 7.3 My choose","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# fold_number = 0\nmarking = pd.read_csv('../input/global-wheat-detection/train.csv')\n# marking = pd.read_csv('../input/check-clean-big-small-bboxes/train_clean.csv')\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:,i]\nmarking.drop(columns=['bbox'], inplace=True)\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ndf_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\ndf_folds.loc[:, 'fold'] = 0\n\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"marking = pd.concat([marking, test_df_pseudo])\ntrain_dataset = DatasetRetriever(\n    image_ids=np.append(df_folds[df_folds['fold'] >1 ].index.values,test_df_pseudo['image_id'].unique()),\n    marking=marking,\n    transforms=get_train_transforms(),\n    test=False,\n)\n\nvalidation_dataset = DatasetRetriever(\n    image_ids=df_folds[df_folds['fold'] <= 1].index.values,\n    marking=marking,\n    transforms=get_valid_transforms(),\n    test=True,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset.image_ids,validation_dataset.image_ids)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 8. Prepare new training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\nclass Fitter:\n    \n    def __init__(self, model, device, config):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{config.folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n        \n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n\n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch(train_loader)\n\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n            t = time.time()\n            summary_loss = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                print(\"better model find at epoch:{}\".format(self.epoch))\n                best_weight=self.model.model.state_dict()\n                self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n        return best_weight\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                images = torch.stack(images)\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                boxes = [target['boxes'].to(self.device).float() for target in targets]\n                labels = [target['labels'].to(self.device).float() for target in targets]\n\n                loss, _, _ = self.model(images, boxes, labels)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        return summary_loss\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n\n            images = torch.stack(images)\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n            boxes = [target['boxes'].to(self.device).float() for target in targets]\n            labels = [target['labels'].to(self.device).float() for target in targets]\n\n            self.optimizer.zero_grad()\n            \n            loss, _, _ = self.model(images, boxes, labels)\n            \n            loss.backward()\n\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss\n    \n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers = 4\n    batch_size = 4\n    n_epochs = 25\n    lr = 0.0001\n\n    folder = 'effdet5-cutmix-augmix'\n\n    verbose = True\n    verbose_step = 1\n\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=1,\n        verbose=False,\n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0,\n        min_lr=1e-8,\n        eps=1e-08\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_training(train_dataset,validation_dataset,train_net=None):\n    device = torch.device('cuda:0')\n    train_net.to(device)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        sampler=RandomSampler(train_dataset),\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n        collate_fn=collate_fn,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_fn,\n    )\n\n    fitter = Fitter(model=train_net, device=device, config=TrainGlobalConfig)\n    best_weight = fitter.fit(train_loader, val_loader)\n    return best_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_net(eval_net):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    # checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(eval_net.model.state_dict())\n\n    # del checkpoint\n    gc.collect()\n\n    net = DetBenchTrain(net, config)\n    net = net.train()\n    return net.cuda()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_net(best_weigth):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    # checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(best_weigth)\n\n    # del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net = net.eval()\n    return net.cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 9. N round Pseudo labeling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom datetime import datetime\nimport time\nRound_N = 1\ntest_net = nets[0]\nPSEUDO_LABELING = len(os.listdir('../input/global-wheat-detection/test/'))>11\nif PSEUDO_LABELING:\n    for i in range(Round_N):\n        train_net = get_train_net(test_net)\n    #     test_df_pseudo = predict_test(data_loader=test_loader,net=test_net)\n    #     train_df = generate_train(test_df_pseudo)\n    #     train_dataset,validation_dataset = split_train_val(train_df)\n        best_weight = run_training(train_dataset=train_dataset,validation_dataset=validation_dataset,train_net=train_net)\n        new_test_net = get_test_net(best_weight)\n        test_net = new_test_net","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###    9.1 twist score_threshold of make_tta_prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_loader = torch.utils.data.DataLoader(\n        validation_dataset,\n        batch_size=4,\n        num_workers=4,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_fn,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_det(index,det,transform,score_threshold=0):\n    boxes = det[index].detach().cpu().numpy()[:,:4]    \n    scores = det[index].detach().cpu().numpy()[:,4]\n    indexes = np.where(scores > score_threshold)[0]\n    boxes = boxes[indexes]\n    boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n    boxes = transform.deaugment_boxes(boxes.copy())\n    boxes = (boxes*2).clip(min=0, max=1023).astype(int)\n    scores = scores[indexes]\n    return boxes, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate(model=None,score_threshold=0): #like make_tta_prediction,score_threshold=0\n    all_predictions = []\n    assert model is not None,\"No model\"\n    \n    for images, targets, image_ids in tqdm(validation_loader, total=len(validation_loader)):\n         with torch.no_grad():\n            images = torch.stack(images)\n            images = images.cuda().float()\n\n            transforms_predictions = {}\n            for transform_number in range(len(tta_transforms)):\n                transforms_predictions[transform_number] = model(tta_transforms[transform_number].batch_augment(images.clone()), torch.tensor([1]*images.shape[0]).float().cuda())\n            \n            for i in range(images.shape[0]):\n                image_predictions = {\n                    'image_id': image_ids[i],\n                    'gt_boxes': (targets[i]['boxes'].cpu().numpy()*2).clip(min=0, max=1023).astype(int)[:, [1, 0, 3, 2]],#target['boxes'][:, [0, 1, 2, 3]] = target['boxes'][:, [1, 0, 3, 2]]\n                }\n                for transform_number in range(len(tta_transforms)):\n                    boxes, scores = process_det(i, transforms_predictions[transform_number],tta_transforms[transform_number],score_threshold)\n                    image_predictions[f'pred_boxes_transform{transform_number}'] = boxes\n                    image_predictions[f'scores_transform{transform_number}'] = scores\n\n                all_predictions.append(image_predictions)\n    return all_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_final_score(all_predictions, score_threshold):\n    final_scores = []\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        image_id = all_predictions[i]['image_id']\n        \n\n        for transform_number in range(len(tta_transforms)):\n            pred_boxes = all_predictions[i][f'pred_boxes_transform{transform_number}'].copy()\n            scores = all_predictions[i][f'scores_transform{transform_number}'].copy()\n            \n            indexes = np.where(scores>score_threshold)\n            pred_boxes = pred_boxes[indexes]\n            scores = scores[indexes]\n            \n            # bugfix: conf should be descending\n            rank = np.argsort(scores)[::-1]\n            scores = scores[rank]\n            pred_boxes = pred_boxes[rank]\n\n            \n            image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')\n            final_scores.append(image_precision)\n\n    return np.mean(final_scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def search_best_score_threshold(all_predictions):\n    best_final_score, best_score_threshold = 0, 0\n    for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01).shape[0]):\n#         print(score_threshold)\n        final_score = calculate_final_score(all_predictions, score_threshold)\n        if final_score > best_final_score:\n            best_final_score = final_score\n            best_score_threshold = score_threshold\n    \n    print('-'*30)\n    print(f'[Best Score Threshold]: {best_score_threshold}')\n    print(f'[OOF Score]: {best_final_score:.4f}')\n    print('-'*30)\n    return best_score_threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if SCORE_OPTIM:\n    all_predictions = validate(model=test_net)\n    \n    #a demo\n\n    gt_boxes = all_predictions[0]['gt_boxes'].copy()\n    pred_boxes = all_predictions[0]['pred_boxes_transform1'].copy()\n    scores = all_predictions[0]['scores_transform1'].copy()\n    image_id = all_predictions[0]['image_id']\n\n    indexes = np.where(scores>best_score_threshold)\n    pred_boxes = pred_boxes[indexes]\n    scores = scores[indexes]\n    show_result(image_id, pred_boxes, gt_boxes)\n\n    \n    best_score_threshold = search_best_score_threshold(all_predictions)\n# else:\n#     best_score_threshold = 0.39","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(best_score_threshold)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 9.2 twist iou_threshold and skip_box_threshold of WBF","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_final_score_wbf(\n    all_predictions,\n    iou_thr,\n    skip_box_thr,\n    method, # weighted_boxes_fusion, nms, soft_nms, non_maximum_weighted\n    sigma=0.5,\n):\n    final_scores = []\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        image_id = all_predictions[i]['image_id']\n        folds_boxes, folds_scores, folds_labels = [], [], []\n        for transform_number in range(len(tta_transforms)):\n            pred_boxes = all_predictions[i][f'pred_boxes_transform{transform_number}'].copy()\n            scores = all_predictions[i][f'scores_transform{transform_number}'].copy()\n            folds_boxes.append(pred_boxes)\n            folds_scores.append(scores)\n            folds_labels.append(np.ones(pred_boxes.shape[0]))\n        \n        if method == 'weighted_boxes_fusion':\n            boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n        elif method == 'nms':\n            boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)\n        elif method == 'soft_nms':\n            boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)\n        elif method == 'non_maximum_weighted':\n            boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n        else:\n            raise\n        image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')\n        final_scores.append(image_precision)\n\n    return np.mean(final_scores)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Bayesian optimization using Gaussian Processes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def log(text):\n    with open('opt.log', 'a+') as logger:\n        logger.write(f'{text}\\n')\n\ndef optimize(space, all_predictions, method, n_calls=10):\n    @use_named_args(space)\n    def score(**params):\n        log('-'*5 + f'{method}' + '-'*5)\n        log(params)\n        final_score = calculate_final_score_wbf(all_predictions, method=method, **params)\n        log(f'final_score = {final_score}')\n        log('-'*10)\n        return -final_score\n\n    return gp_minimize(func=score, dimensions=space, n_calls=n_calls)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ensemble_boxes import *\nif WBF_OPTIM:\n    all_predictions = validate(model=test_net,score_threshold=best_score_threshold)\n\n    #a demo\n\n    gt_boxes = all_predictions[0]['gt_boxes'].copy()\n    pred_boxes = all_predictions[0]['pred_boxes_transform1'].copy()\n    scores = all_predictions[0]['scores_transform1'].copy()\n    image_id = all_predictions[0]['image_id']\n\n    indexes = np.where(scores>best_score_threshold)\n    pred_boxes = pred_boxes[indexes]\n    scores = scores[indexes]\n    show_result(image_id, pred_boxes, gt_boxes)\n    \n    \n    space = [\n        Real(0, 1, name='iou_thr'),\n        Real(0.25, 1, name='skip_box_thr'),\n    ]\n\n    opt_result = optimize(\n        space, \n        all_predictions,\n        method='weighted_boxes_fusion',\n        n_calls=50,\n    )\n\n    best_final_score = -opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\n\n\n    print('-'*13 + 'WBF' + '-'*14)\n    print(f'[Best Iou Thr]: {best_iou_thr:.3f}')\n    print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\n    print(f'[Best Score]: {best_final_score:.4f}')\n    print('-'*30)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(best_iou_thr,best_skip_box_thr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 10. inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(best_skip_box_thr)\nresults = []\nfig, ax = plt.subplots(5, 2, figsize=(30, 70))\ncount = 0\nfor images, image_ids in test_data_loader:\n    predictions = make_tta_predictions(images,score_threshold=best_score_threshold,nets=[test_net])\n    for i, image in enumerate(images):\n        boxes, scores, labels = run_wbf(predictions, image_index=i, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)\n        boxes = (boxes * 2).astype(np.int32).clip(min=0, max=1023)\n        \n#         boxes = boxes[scores >0.27 ].astype(np.int32)\n#         scores = scores[scores >0.27]\n            \n\n        image_id = image_ids[i]\n\n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n        if count<10:\n            sample = cv2.resize(image.permute(1,2,0).cpu().numpy(),(1024,1024))\n            for box, score in zip(boxes,scores):\n                cv2.rectangle(sample,\n                              (box[0], box[1]),\n                              (box[2]+box[0], box[3]+box[1]),\n                              (220, 0, 0), 2)\n                cv2.putText(sample, '%.2f'%(score), (box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX ,  \n                   0.5, (255,255,255), 2, cv2.LINE_AA)\n            ax[count%5][count//5].imshow(sample)\n            count+=1\n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n        results.append(result)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}