{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference Kernel on Test Data\n\n# 1. Prepare ResNest and Best Weights Data\nJung's already make ResNest repo as a dataset. We will move this to Kaggle's working directory so that we can modify (`/kaggle/input/` is read-only)","metadata":{}},{"cell_type":"code","source":"!cp -rf /kaggle/input/wheatdetection-resnest-develop-branch-july9/wheatdetection/wheatdetection /kaggle/working/\n!ls /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -rf /kaggle/input/mydata /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CODE_PATH = '/kaggle/working/wheatdetection/'\n!ls {CODE_PATH}\n%cd {CODE_PATH}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we define a path to our best checkpoint. Please modify the following cell to your own trained dataset.","metadata":{}},{"cell_type":"code","source":"import torch\nBEST_PATHS = ['/kaggle/input/mydata/FRCNN_F0_last.bin',\n             '/kaggle/input/mydata/FRCNN_F1_last.bin',\n              '/kaggle/input/mydata/FRCNN_F2_last.bin',]\n\n# print to see best weights information\nfor BEST_PATH in BEST_PATHS:\n    ckp = torch.load(BEST_PATH)\n    print(ckp.keys())\n    print(ckp['epoch'], ckp['best_valid_loss'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Inference\n\nDefine hyperparameters here. If we don't use NMS, we simply filter out the boxes using `SCORE_THRESHOLD` ; If we use NMS, we will combine both `SCORE_THRESHOLD` and `IOU_THRESHOLD` to filter box according to NMS logic.","metadata":{}},{"cell_type":"code","source":"USE_NMS = False\nSCORE_THRESHOLD = 0.6\nNMS_IOU_THRESHOLD = 0.5\nIMG_SIZE = 1024\nN_VIZ = 10 \nWBF_SCORE_THRESHOLD = 0.25\nUSE_BOUNDS_FILTER = True \nLOWER_BOUND, UPPER_BOUND = 70, 175000 # observed bounds from Train data (400, 150000)\nPP_SHRINK = [0,0]\nWBF_IOU, WBF_SKIP_BOX = 0.4, 0.34","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the repo, WheatDetector Class always need internet connection for `pretrained` weights, so below we rewrite the file for `pretrained=False`","metadata":{}},{"cell_type":"code","source":"%%writefile ./data/build.py\n# encoding: utf-8\n\"\"\"\n@author:  wuxin.wang\n@contact: wuxin.wang@whu.edu.cn\n\"\"\"\n\nfrom torch.utils import data\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom .datasets.train_wheat import train_wheat\nfrom .datasets.test_wheat import test_wheat\nfrom .transforms import build_transforms\nfrom .transforms import get_test_transform\nfrom .collate_batch import collate_batch\n\ndef split_dataset(cfg):\n    marking = pd.read_csv(f'{cfg.DATASETS.DIR}')\n    df = marking.groupby('image_id').count()\n#     bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n#     for i, column in enumerate(['x', 'y', 'w', 'h']):\n#         marking[column] = bboxs[:, i]\n#     marking.drop(columns=['bbox'], inplace=True)\n#     marking['area'] = marking['w'] * marking['h']\n#     marking = marking[marking['area'] < 154200.0]\n#     error_bbox = [100648.0, 145360.0, 149744.0, 119790.0, 106743.0]\n#     marking = marking[~marking['area'].isin(error_bbox)]\n#     marking = marking[marking['h']>16.0]\n#     marking = marking[marking['w']>16.0]\n\n#     skf = StratifiedKFold(n_splits=1, shuffle=True, random_state=42)\n\n#     df_folds = marking[['image_id']].copy()\n#     df_folds.loc[:, 'bbox_count'] = 1\n#     df_folds = df_folds.groupby('image_id').count()\n#     df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n#     df_folds.loc[:, 'stratify_group'] = np.char.add(\n#         df_folds['source'].values.astype(str),\n#         df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n#     )\n#     df_folds.loc[:, 'fold'] = 0\n\n#     for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n#         df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n\n#     train_ids = df_folds[df_folds['fold'] != cfg.DATASETS.VALID_FOLD].index.values\n    valid_ids = df.index.values\n#     if cfg.DEBUG:\n#         train_ids = train_ids[:40]\n#         valid_ids = valid_ids[:10]\n\n    return marking, valid_ids\n\ndef build_dataset(cfg):\n    marking, valid_ids = split_dataset(cfg)\n#     train_dataset = train_wheat(\n#         root = cfg.DATASETS.ROOT_DIR,\n#         image_ids=train_ids,\n#         marking=marking,\n#         transforms=build_transforms(cfg, is_train=True),\n#         test=False,\n#     )\n\n    validation_dataset = train_wheat(\n        root=cfg.DATASETS.ROOT_DIR,\n        image_ids=valid_ids,\n        marking=marking,\n        transforms=build_transforms(cfg, is_train=False),\n        test=True,\n    )\n\n    return validation_dataset\n\ndef make_data_loader(cfg, is_train=True):\n    if is_train:\n        batch_size = cfg.SOLVER.IMS_PER_BATCH\n    else:\n        batch_size = cfg.TEST.IMS_PER_BATCH\n\n    validation_dataset = build_dataset(cfg)\n\n    num_workers = cfg.DATALOADER.NUM_WORKERS\n#     train_loader = data.DataLoader(\n#         train_dataset,\n#         batch_size=batch_size,\n#         sampler=RandomSampler(train_dataset),\n#         pin_memory=False,\n#         drop_last=True,\n#         num_workers=num_workers,\n#         collate_fn=collate_batch,\n#     )\n    val_loader = data.DataLoader(\n        validation_dataset,\n        batch_size=batch_size,\n        num_workers=num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_batch,\n    )\n\n    return val_loader\n\ndef build_test_dataset(cfg):\n    test_df = pd.read_csv(f'{cfg.DATASETS.ROOT_DIR}/sample_submission.csv')\n    print(test_df.shape)\n    test_dataset = test_wheat(test_df, f'{cfg.DATASETS.ROOT_DIR}/test', get_test_transform())\n\n    return test_dataset\n\ndef make_test_data_loader(cfg):\n    batch_size = cfg.TEST.IMS_PER_BATCH\n\n    test_dataset = build_test_dataset(cfg)\n\n    num_workers = cfg.DATALOADER.NUM_WORKERS\n\n    test_loader = data.DataLoader(\n        test_dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=num_workers,\n        drop_last=False,\n        collate_fn=collate_batch\n    )\n\n    return test_loader","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./modeling/wheat_detector.py\n\nimport torch\nfrom torch import nn\nfrom layers import FasterRCNN\nfrom layers.backbone_utils import resnest_fpn_backbone\n\nclass WheatDetector(nn.Module):\n    def __init__(self, cfg, **kwargs):\n        super(WheatDetector, self).__init__()\n        self.backbone = resnest_fpn_backbone(pretrained=False) #change here\n        self.base = FasterRCNN(self.backbone, num_classes=cfg.MODEL.NUM_CLASSES, **kwargs)\n\n    def forward(self, images, targets=None):\n        return self.base(images, targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./config/defaults.py\n# encoding: utf-8\n\"\"\"\n@author:  wuxin.wang\n@contact: wuxin.wang@whu.edu.cn\n\"\"\"\nimport sys\nsys.path.insert(0, \"./external/yacs\")\nfrom yacs.config import CfgNode as CN\n\n# -----------------------------------------------------------------------------\n# Convention about Training / Test specific parameters\n# -----------------------------------------------------------------------------\n# Whenever an argument can be either used for training or for testing, the\n# corresponding name will be post-fixed by a _TRAIN for a training parameter,\n# or _TEST for a test-specific parameter.\n# For example, the number of images during training will be\n# IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be\n# IMAGES_PER_BATCH_TEST\n\n# -----------------------------------------------------------------------------\n# Config definition\n# -----------------------------------------------------------------------------\n\n_C = CN()\n\n_C.DEBUG = False\n_C.SEED = 42\n_C.VERBOSE = True\n\n_C.MODEL = CN()\n_C.MODEL.DEVICE = \"cuda\"\n_C.MODEL.NUM_CLASSES = 2\n\n# -----------------------------------------------------------------------------\n# INPUT\n# -----------------------------------------------------------------------------\n_C.INPUT = CN()\n# RandomSizedCrop paramters\n_C.INPUT.RSC_MIN_MAX_HEIGHT = (800, 800)\n_C.INPUT.RSC_HEIGHT = 1024\n_C.INPUT.RSC_WIDTH = 1024\n_C.INPUT.RSC_PROB = 0.5\n# HueSaturationValue paramters\n_C.INPUT.HSV_H = 0.2\n_C.INPUT.HSV_S = 0.2\n_C.INPUT.HSV_V = 0.2\n_C.INPUT.HSV_PROB = 0.9\n# RandomBrightnessContrast paramters\n_C.INPUT.BC_B = 0.2\n_C.INPUT.BC_C = 0.2\n_C.INPUT.BC_PROB = 0.9\n# Color paramters\n_C.INPUT.COLOR_PROB = 0.9\n# Random probability for ToGray\n_C.INPUT.TOFGRAY_PROB = 0.01\n# Random probability for HorizontalFlip\n_C.INPUT.HFLIP_PROB = 0.5\n# Random probability for VerticalFlip\n_C.INPUT.VFLIP_PROB = 0.5\n# Coutout paramters\n_C.INPUT.COTOUT_NUM_HOLES = 8\n_C.INPUT.COTOUT_MAX_H_SIZE = 64\n_C.INPUT.COTOUT_MAX_W_SIZE = 64\n_C.INPUT.COTOUT_FILL_VALUE = 0\n_C.INPUT.COTOUT_PROB = 0.5\n\n# -----------------------------------------------------------------------------\n# Dataset\n# -----------------------------------------------------------------------------\n_C.DATASETS = CN()\n# Root dir of dataset\n_C.DATASETS.ROOT_DIR = \"/content/global-wheat-detection\"\n_C.DATASETS.DIR = \"/content/global-wheat-detection\"\n# Fold to validate\n_C.DATASETS.VALID_FOLD = 0\n# # List of the dataset names for training, as present in paths_catalog.py\n# _C.DATASETS.TRAIN = ()\n# # List of the dataset names for testing, as present in paths_catalog.py\n# _C.DATASETS.TEST = ()\n\n# -----------------------------------------------------------------------------\n# DataLoader\n# -----------------------------------------------------------------------------\n_C.DATALOADER = CN()\n# Number of data loading threads\n_C.DATALOADER.NUM_WORKERS = 2\n\n# ---------------------------------------------------------------------------- #\n# Solver\n# ---------------------------------------------------------------------------- #\n_C.SOLVER = CN()\n_C.SOLVER.OPTIMIZER_NAME = \"SGD\"\n_C.SOLVER.SCHEDULER_NAME = \"CosineAnnealingWarmRestarts\"\n_C.SOLVER.COS_CPOCH = 2\n_C.SOLVER.T_MUL = 2\n\n_C.SOLVER.MAX_EPOCHS = 72\n\n_C.SOLVER.BASE_LR = 0.005\n_C.SOLVER.BIAS_LR_FACTOR = 1\n\n_C.SOLVER.MOMENTUM = 0.9\n\n_C.SOLVER.WEIGHT_DECAY = 0.0005\n_C.SOLVER.WEIGHT_DECAY_BIAS = 0\n_C.SOLVER.WEIGHT_DECAY_BN = 0\n\n_C.SOLVER.WARMUP_EPOCHS = 10\n\n_C.SOLVER.EARLY_STOP_PATIENCE = 20\n\n_C.SOLVER.TRAIN_CHECKPOINT = False\n_C.SOLVER.CLEAR_OUTPUT = True\n\n# Number of images per batch\n# This is global, so if we have 8 GPUs and IMS_PER_BATCH = 16, each GPU will\n# see 2 images per batch\n_C.SOLVER.IMS_PER_BATCH = 4\n\n# This is global, so if we have 8 GPUs and IMS_PER_BATCH = 16, each GPU will\n# see 2 images per batch\n_C.TEST = CN()\n_C.TEST.IMS_PER_BATCH = 4\n_C.TEST.WEIGHT = \"/content/output/best-checkpoint.bin\"\n\n# ---------------------------------------------------------------------------- #\n# Misc options\n# ---------------------------------------------------------------------------- #\n_C.OUTPUT_DIR = \"/content/drive/My Drive/Global_Wheat_Detection/experiments/baseline\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define TTA (Hide cell)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\n\nimport os\nimport warnings\nimport torch\nimport numpy as np\nfrom tqdm import tqdm\nimport pandas as pd\nfrom itertools import product\nimport sys\nsys.path.insert(0, \"./external/wbf\")\nimport ensemble_boxes\nwarnings.filterwarnings(\"ignore\")\n\n\nclass BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n\n    def batch_augment(self, images):\n        raise NotImplementedError\n\n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n\n    def batch_augment(self, images):\n        return images.flip(2)\n\n    def deaugment_boxes(self, boxes):\n        boxes[:, [1, 3]] = self.image_size - boxes[:, [3, 1]]\n        return boxes\n\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(2)\n\n    def batch_augment(self, images):\n        return images.flip(3)\n\n    def deaugment_boxes(self, boxes):\n        boxes[:, [0, 2]] = self.image_size - boxes[:, [2, 0]]\n        return boxes\n\n\nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n\n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0, 2]] = self.image_size - boxes[:, [3, 1]]\n        res_boxes[:, [1, 3]] = boxes[:, [0, 2]]\n        return res_boxes\n\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def __init__(self, transforms):\n        self.transforms = transforms\n\n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n\n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n\n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:, 0] = np.min(boxes[:, [0, 2]], axis=1)\n        result_boxes[:, 2] = np.max(boxes[:, [0, 2]], axis=1)\n        result_boxes[:, 1] = np.min(boxes[:, [1, 3]], axis=1)\n        result_boxes[:, 3] = np.max(boxes[:, [1, 3]], axis=1)\n        return result_boxes\n\n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from evaluate.evaluate import evaluate\n\n# Tester is modified for multi-models\nclass Tester:\n    def __init__(self, models, device, cfg, test_loader, n_viz=N_VIZ):\n        self.config = cfg\n        self.test_loader = test_loader\n\n        self.base_dir = f'{self.config.OUTPUT_DIR}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.score_threshold = SCORE_THRESHOLD\n        self.iou_threshold = NMS_IOU_THRESHOLD\n        self.use_nms = USE_NMS\n        self.n_viz = n_viz\n        \n        self.device = device\n        self.models = []\n        for i,model in enumerate(models):\n            self.models.append(model)\n            self.models[-1].eval()\n            self.models[-1].to(self.device)\n\n        self.log(f'Tester prepared. Device is {self.device}')\n\n    def test(self, pp_shrink=0):\n        results,all_predictions = self.infer(pp_shrink)\n        self.save_predictions(results)\n        return all_predictions\n    def process_det(self, index, outputs):\n        boxes = outputs[index]['boxes'].data.cpu().numpy()\n        scores = outputs[index]['scores'].data.cpu().numpy()\n        boxes = (boxes).clip(min=0, max=1023).astype(int)\n        indexes = np.where(scores > self.score_threshold)\n        boxes = boxes[indexes]\n        scores = scores[indexes]\n        return boxes, scores\n\n    def make_tta_predictions(self, tta_transforms, images, model_id):\n        with torch.no_grad():\n            images = torch.stack(images).float().cuda()\n            predictions = []\n            for tta_transform in tta_transforms:\n                result = []\n                outputs = self.models[model_id](tta_transform.batch_augment(images.clone()))\n                \n                \n                for i, image in enumerate(images):\n                    boxes = outputs[i]['boxes'].data.cpu().numpy()\n                    scores = outputs[i]['scores'].data.cpu().numpy()\n                    indexes = np.where(scores > self.score_threshold)[0]\n                    boxes = tta_transform.deaugment_boxes(boxes.copy())\n                    \n                    if self.use_nms: \n                        labels = np.ones(scores.shape[0]).astype(int).tolist()\n                        boxes, scores, labels = ensemble_boxes.ensemble_boxes_nms.nms_method([boxes], [scores], [labels], method=3,\n                                                                                        weights=None, iou_thr=self.iou_threshold,\n                                                                                        thresh=self.score_threshold)\n                    else: # not use NMS, just filter by confidence score\n                        boxes = boxes[indexes]\n                        scores = scores[indexes]\n                    result.append({\n                        'boxes': boxes,\n                        'scores': scores,\n                    })\n                predictions.append(result)\n        return predictions # [TTA_NUM , BATCH_IMG_NUM, DICT[BOXES, SCORES]]\n    \n    '''\n    run_wbf\n    - predictions is in [TTA_NUM , BATCH_IMG_NUM, DICT[BOXES, SCORES]] format\n    - BOXES are xyxy format \n    - BOXES are unnormalized -- and will get WRONG result for non-square non-1024 images!! (hidden BUG FOUND)\n    '''\n    def run_wbf(self, predictions, image_index, image_shape=(IMG_SIZE,IMG_SIZE), iou_thr=WBF_IOU, skip_box_thr=WBF_SKIP_BOX, weights=None):\n#         boxes = [(prediction[image_index]['boxes'] / (image_size - 1)).tolist() for prediction in predictions] # for each TTA\n        \n        new_boxes_multi_models = []\n        for prediction in predictions:\n            pred_boxes = prediction[image_index]['boxes'] # each (TTA) model, specific image_id --> np-shape[NUM_BOXES, 4] \n            pred_boxes[:, 0] = pred_boxes[:, 0]/(image_shape[0]-1)\n            pred_boxes[:, 1] = pred_boxes[:, 1]/(image_shape[1]-1)\n            pred_boxes[:, 2] = pred_boxes[:, 2]/(image_shape[0]-1)\n            pred_boxes[:, 3] = pred_boxes[:, 3]/(image_shape[1]-1)\n            \n            new_boxes_multi_models.append(pred_boxes.tolist())\n        \n        boxes = new_boxes_multi_models\n        scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n        labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in\n                  predictions]\n        boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels,\n                                                                                        weights=None, iou_thr=iou_thr,\n                                                                                        skip_box_thr=skip_box_thr)\n#         boxes = boxes * (image_size - 1)\n        boxes[:, 0] = boxes[:, 0]*(image_shape[0]-1)\n        boxes[:, 1] = boxes[:, 1]*(image_shape[1]-1)\n        boxes[:, 2] = boxes[:, 2]*(image_shape[0]-1)\n        boxes[:, 3] = boxes[:, 3]*(image_shape[1]-1)\n        return boxes, scores, labels\n\n    def format_prediction_string(self, boxes, scores):\n        pred_strings = []\n        for j in zip(scores, boxes):\n            pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n        return \" \".join(pred_strings)\n\n    def infer(self, pp_shrink=0):\n        for i in range(len(self.models)):\n            self.models[i].eval()\n        torch.cuda.empty_cache()\n\n        tta_transforms = []\n        for tta_combination in product([TTAHorizontalFlip(), None],\n                                       [TTAVerticalFlip(), None],\n#                                        [TTAReduceSaturation(), None],\n                                       [TTARotate90(), None]):\n            tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))\n        test_loader = tqdm(self.test_loader, total=len(self.test_loader), desc=\"Testing\")\n        results = []\n        all_predictions = []\n        boxes10 = []\n        \n        for images, targets, image_ids in test_loader:\n            predictions=[]\n            for ii in range(len(self.models)):\n                predictions.append(self.make_tta_predictions(tta_transforms, images,model_id=ii)) # [TTA_NUM , BATCH_IMG_NUM, DICT[BOXES, SCORES]]\n            \n            predictions = np.vstack(predictions)\n            \n            for i, image in enumerate(images):\n#                 print('shape : ',image.shape)\n                boxes, scores, labels = self.run_wbf(predictions, image_index=i,image_shape=[image.shape[1],image.shape[2]])\n\n                #round and clip seems to be better before rather than after pp_shrink\n                boxes = boxes.round().astype(np.int32).clip(min=0, max=1023)\n                \n                image_id = image_ids[i]\n                \n                if len(boxes10) < self.n_viz:\n                    print('writing ... ',i,image_id)\n                    sample = image.permute(1,2,0).cpu().numpy()\n\n                    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n                    boxes10.append((sample, boxes))\n                    \n                    for box, score in zip(boxes,scores):\n                        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (0, 0, 1), 4)\n                        cv2.putText(sample, '%.2f'%(score), (box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX ,  \n                   1, (255,255,255), 3, cv2.LINE_AA)\n                    \n                    ax.set_axis_off()\n                    ax.imshow(sample);\n                    plt.show()\n                                \n                #post-processing box size adjusting (host advised boxes in test are tight to image)\n                boxes[:, 0] = boxes[:, 0] + pp_shrink\n                boxes[:, 1] = boxes[:, 1] + pp_shrink\n                boxes[:, 2] = boxes[:, 2] - pp_shrink \n                boxes[:, 3] = boxes[:, 3] - pp_shrink \n                \n                img_shape = image.cpu().numpy().shape\n                boxes[:, 0] = [max(min(x, img_shape[1]-1), 0) for x in boxes[:, 0]]\n                boxes[:, 1] = [max(min(x, img_shape[2]-1), 0) for x in boxes[:, 1]]\n                boxes[:, 2] = [max(min(x, img_shape[1]-1), 0) for x in boxes[:, 2]]\n                boxes[:, 3] = [max(min(x, img_shape[2]-1), 0) for x in boxes[:, 3]]\n                \n                boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n                \n                areas = boxes[:, 2]*boxes[:, 3]\n                gt = targets[i]['boxes'].cpu().numpy().astype(int)\n                gt[:, 2] = gt[:, 2] - gt[:, 0]\n                gt[:, 3] = gt[:, 3] - gt[:, 1]\n#                 if USE_BOUNDS_FILTER==True:\n#                     #if boxes is filtered do we need to filter scores as well? Otherwise the score will not be aligned with the correct box?\n#                     #boxes = [boxes[i] for i in range(len(boxes)) if areas[i] > LOWER_BOUND and areas[i] < UPPER_BOUND]\n#                     print(\"Filter by area bounds\")\n#                     print(\"Length of unfiltered boxes: \" + str(len(boxes)))\n#                     print(\"Length of unfiltered scores: \" + str(len(scores)))\n#                     filtered_boxes = []\n#                     filtered_scores = []\n#                     for i in range(len(boxes)):\n#                         if areas[i] > LOWER_BOUND and areas[i] < UPPER_BOUND and scores[i] >= WBF_SCORE_THRESHOLD:\n#                             filtered_boxes.append(boxes[i])\n#                             filtered_scores.append(scores[i])\n#                     print(\"Length of filtered boxes: \" + str(len(filtered_boxes)))\n#                     print(\"Length of filtered scores: \" + str(len(filtered_scores)))\n#                     boxes = filtered_boxes\n#                     scores = filtered_scores\n#                 else:\n#                     print(\"area filtering not applied\")\n                        \n                \n                result = {\n                    'image_id': image_id,\n                    'PredictionString': self.format_prediction_string(boxes, scores),\n                }\n                \n                all_prediction = {\n                    'pred_boxes': boxes,\n                    'scores': scores,\n                    'gt_boxes': gt,\n                    'image_id': image_id,\n                }\n                \n                results.append(result)\n                all_predictions.append(all_prediction)\n        return results, all_predictions\n\n    def format_prediction_string(self, boxes, scores):\n        pred_strings = []\n        for j in zip(scores, boxes):\n            pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n        return \" \".join(pred_strings)\n\n    def save_predictions(self, results):\n        test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\n        test_df.to_csv(f'{self.config.OUTPUT_DIR}/submission.csv', index=False)\n\n    def load(self, paths):\n        print(paths)\n        for i in range(len(paths)):\n            checkpoint = torch.load(paths[i])\n            self.models[i].load_state_dict(checkpoint['model_state_dict'])\n            print('finish loading ',paths[i])\n    def log(self, message):\n        if self.config.VERBOSE:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split fold and make correct CV with TTA/WBF/NMS\nSplit valid fold : specify fold in `cfg.DATASETS.VALID_FOLD`","metadata":{}},{"cell_type":"code","source":"from config import cfg\n\ncfg['OUTPUT_DIR'] = \"/kaggle/working/\"\ncfg['DATASETS']['ROOT_DIR'] = \"/kaggle/input/global-wheat-detection\"\ncfg['DATASETS']['DIR'] = \"/kaggle/input/mydata/test.csv\"\ncfg['TEST']['WEIGHT'] = BEST_PATHS\ncfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from data import make_data_loader\nval_loader = make_data_loader(cfg, is_train=False)\n# test_loader = make_test_data_loader(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\n\nfrom os import mkdir\nsys.path.append('.')\nfrom config import cfg\nfrom data import make_test_data_loader\nfrom modeling import build_model\nfrom utils.logger import setup_logger\n\n# start here!!\nif True:\n#     cfg.freeze()\n\n    output_dir = cfg.OUTPUT_DIR\n    if output_dir and not os.path.exists(output_dir):\n        print('creating ',cfg.OUTPUT_DIR)\n        mkdir(output_dir)\n    models=[]\n    for i, best in enumerate(BEST_PATHS):\n        print(i,'---',best)\n        models.append(build_model(cfg)) # build_model uses cfg only on cfg.num_classes    \n    \ndevice = cfg.MODEL.DEVICE\ntester = Tester(models=models, device=device, cfg=cfg, test_loader=val_loader, n_viz=N_VIZ)\ntester.load(cfg['TEST']['WEIGHT'])\nprint('*** success load weights! ***')    \nall_preds = tester.test(pp_shrink=PP_SHRINK[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numba\nfrom evaluate.calculate_score import *\ndef calculate_individual_scores(all_predictions, score_threshold):\n    final_scores = []\n    final_missed_boxes_nums = []\n    image_id_list = []\n    # Numba typed list!\n    iou_thresholds = numba.typed.List()\n\n    for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n        iou_thresholds.append(x)\n\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        pred_boxes = all_predictions[i]['pred_boxes'].copy()\n        scores = all_predictions[i]['scores'].copy()\n        image_id = all_predictions[i]['image_id']\n        \n        indexes = np.where(scores > score_threshold)\n        pred_boxes = pred_boxes[indexes]\n        scores = scores[indexes]\n        \n        image_precision = calculate_image_precision(gt_boxes, pred_boxes, thresholds=iou_thresholds, \n                                                    form='coco'#'pascal_voc'\n                                                   )\n        final_scores.append(image_precision)\n        image_id_list.append(image_id)\n    return final_scores, image_id_list, np.mean(final_scores)\n\ndef evaluate(all_predictions):\n    best_final_score, best_score_threshold = 0, 0\n    for score_threshold in tqdm(np.arange(0, 1, 0.05), total=np.arange(0, 1, 0.05).shape[0], desc=\"OOF\"):\n        _,_,final_score = calculate_individual_scores(all_predictions, score_threshold)\n        if final_score > best_final_score:\n            best_final_score = final_score\n            best_score_threshold = score_threshold\n\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        pred_boxes = all_predictions[i]['pred_boxes'].copy()\n        scores = all_predictions[i]['scores'].copy()\n        indexes = np.where(scores>best_score_threshold)\n        pred_boxes = pred_boxes[indexes]\n        all_predictions[i]['final_missed_boxes_nums'] = len(gt_boxes)-len(pred_boxes)\n\n    return best_score_threshold, best_final_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correct CV is printed out here!\nThis CV taking into account TTA and WBF","metadata":{}},{"cell_type":"code","source":"best_score_threshold, best_final_score = evaluate(all_preds)\n\nprint('CV -- mAP .5:.75 for valid inference with best threshold = %.2f, TTA/WBF/NMS = %.4f' % (best_score_threshold, best_final_score))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}