{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference Kernel on Test Data\n\n# 1. Prepare ResNest and Best Weights Data\nJung's already make ResNest repo as a dataset. We will move this to Kaggle's working directory so that we can modify (`/kaggle/input/` is read-only)","metadata":{}},{"cell_type":"code","source":"!cp -rf /kaggle/input/wheatdetection-resnest-develop-branch-july9/wheatdetection/wheatdetection /kaggle/working/\n!ls /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -rf /kaggle/input/mydata /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CODE_PATH = '/kaggle/working/wheatdetection/'\n!ls {CODE_PATH}\n%cd {CODE_PATH}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we define a path to our best checkpoint. Please modify the following cell to your own trained dataset.","metadata":{}},{"cell_type":"code","source":"import torch\nBEST_PATH = \"/kaggle/input/mydata/FRCNN_F0_best.bin\"\n\n# print to see best weights information../input/gdrive-dataset-downloader/F0_86_finetune_epoch_best.bin\nckp = torch.load(BEST_PATH)\nprint(ckp.keys())\nprint(ckp['epoch'], ckp['best_valid_loss'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Inference\n\nDefine hyperparameters here. If we don't use NMS, we simply filter out the boxes using `SCORE_THRESHOLD` ; If we use NMS, we will combine both `SCORE_THRESHOLD` and `IOU_THRESHOLD` to filter box according to NMS logic.","metadata":{}},{"cell_type":"code","source":"USE_NMS = False\nSCORE_THRESHOLD = 0.8\nNMS_IOU_THRESHOLD = 0.5\nIMG_SIZE = 1024\n\nWBF_IOU, WBF_SKIP_BOX = 0.8, 0.8","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the repo, WheatDetector Class always need internet connection for `pretrained` weights, so below we rewrite the file for `pretrained=False`","metadata":{}},{"cell_type":"code","source":"%%writefile ./data/build.py\n# encoding: utf-8\n\"\"\"\n@author:  wuxin.wang\n@contact: wuxin.wang@whu.edu.cn\n\"\"\"\n\nfrom torch.utils import data\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom .datasets.train_wheat import train_wheat\nfrom .datasets.test_wheat import test_wheat\nfrom .transforms import build_transforms\nfrom .transforms import get_test_transform\nfrom .collate_batch import collate_batch\n\ndef split_dataset(cfg):\n    marking = pd.read_csv(f'{cfg.DATASETS.DIR}')\n    df = marking.groupby('image_id').count()\n#     bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n#     for i, column in enumerate(['x', 'y', 'w', 'h']):\n#         marking[column] = bboxs[:, i]\n#     marking.drop(columns=['bbox'], inplace=True)\n#     marking['area'] = marking['w'] * marking['h']\n#     marking = marking[marking['area'] < 154200.0]\n#     error_bbox = [100648.0, 145360.0, 149744.0, 119790.0, 106743.0]\n#     marking = marking[~marking['area'].isin(error_bbox)]\n#     marking = marking[marking['h']>16.0]\n#     marking = marking[marking['w']>16.0]\n\n#     skf = StratifiedKFold(n_splits=1, shuffle=True, random_state=42)\n\n#     df_folds = marking[['image_id']].copy()\n#     df_folds.loc[:, 'bbox_count'] = 1\n#     df_folds = df_folds.groupby('image_id').count()\n#     df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n#     df_folds.loc[:, 'stratify_group'] = np.char.add(\n#         df_folds['source'].values.astype(str),\n#         df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n#     )\n#     df_folds.loc[:, 'fold'] = 0\n\n#     for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n#         df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n\n#     train_ids = df_folds[df_folds['fold'] != cfg.DATASETS.VALID_FOLD].index.values\n    valid_ids = df.index.values\n#     if cfg.DEBUG:\n#         train_ids = train_ids[:40]\n#         valid_ids = valid_ids[:10]\n\n    return marking, valid_ids\n\ndef build_dataset(cfg):\n    marking, valid_ids = split_dataset(cfg)\n#     train_dataset = train_wheat(\n#         root = cfg.DATASETS.ROOT_DIR,\n#         image_ids=train_ids,\n#         marking=marking,\n#         transforms=build_transforms(cfg, is_train=True),\n#         test=False,\n#     )\n\n    validation_dataset = train_wheat(\n        root=cfg.DATASETS.ROOT_DIR,\n        image_ids=valid_ids,\n        marking=marking,\n        transforms=build_transforms(cfg, is_train=False),\n        test=True,\n    )\n\n    return validation_dataset\n\ndef make_data_loader(cfg, is_train=True):\n    if is_train:\n        batch_size = cfg.SOLVER.IMS_PER_BATCH\n    else:\n        batch_size = cfg.TEST.IMS_PER_BATCH\n\n    validation_dataset = build_dataset(cfg)\n\n    num_workers = cfg.DATALOADER.NUM_WORKERS\n#     train_loader = data.DataLoader(\n#         train_dataset,\n#         batch_size=batch_size,\n#         sampler=RandomSampler(train_dataset),\n#         pin_memory=False,\n#         drop_last=True,\n#         num_workers=num_workers,\n#         collate_fn=collate_batch,\n#     )\n    val_loader = data.DataLoader(\n        validation_dataset,\n        batch_size=batch_size,\n        num_workers=num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_batch,\n    )\n\n    return val_loader\n\ndef build_test_dataset(cfg):\n    test_df = pd.read_csv(f'{cfg.DATASETS.ROOT_DIR}/sample_submission.csv')\n    print(test_df.shape)\n    test_dataset = test_wheat(test_df, f'{cfg.DATASETS.ROOT_DIR}/test', get_test_transform())\n\n    return test_dataset\n\ndef make_test_data_loader(cfg):\n    batch_size = cfg.TEST.IMS_PER_BATCH\n\n    test_dataset = build_test_dataset(cfg)\n\n    num_workers = cfg.DATALOADER.NUM_WORKERS\n\n    test_loader = data.DataLoader(\n        test_dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=num_workers,\n        drop_last=False,\n        collate_fn=collate_batch\n    )\n\n    return test_loader","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./modeling/wheat_detector.py\n\nimport torch\nfrom torch import nn\nfrom layers import FasterRCNN\nfrom layers.backbone_utils import resnest_fpn_backbone\n\nclass WheatDetector(nn.Module):\n    def __init__(self, cfg, **kwargs):\n        super(WheatDetector, self).__init__()\n        self.backbone = resnest_fpn_backbone(pretrained=False) #change here\n        self.base = FasterRCNN(self.backbone, num_classes=cfg.MODEL.NUM_CLASSES, **kwargs)\n\n    def forward(self, images, targets=None):\n        return self.base(images, targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./config/defaults.py\n# encoding: utf-8\n\"\"\"\n@author:  wuxin.wang\n@contact: wuxin.wang@whu.edu.cn\n\"\"\"\nimport sys\nsys.path.insert(0, \"./external/yacs\")\nfrom yacs.config import CfgNode as CN\n\n# -----------------------------------------------------------------------------\n# Convention about Training / Test specific parameters\n# -----------------------------------------------------------------------------\n# Whenever an argument can be either used for training or for testing, the\n# corresponding name will be post-fixed by a _TRAIN for a training parameter,\n# or _TEST for a test-specific parameter.\n# For example, the number of images during training will be\n# IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be\n# IMAGES_PER_BATCH_TEST\n\n# -----------------------------------------------------------------------------\n# Config definition\n# -----------------------------------------------------------------------------\n\n_C = CN()\n\n_C.DEBUG = False\n_C.SEED = 42\n_C.VERBOSE = True\n\n_C.MODEL = CN()\n_C.MODEL.DEVICE = \"cuda\"\n_C.MODEL.NUM_CLASSES = 2\n\n# -----------------------------------------------------------------------------\n# INPUT\n# -----------------------------------------------------------------------------\n_C.INPUT = CN()\n# RandomSizedCrop paramters\n_C.INPUT.RSC_MIN_MAX_HEIGHT = (800, 800)\n_C.INPUT.RSC_HEIGHT = 1024\n_C.INPUT.RSC_WIDTH = 1024\n_C.INPUT.RSC_PROB = 0.5\n# HueSaturationValue paramters\n_C.INPUT.HSV_H = 0.2\n_C.INPUT.HSV_S = 0.2\n_C.INPUT.HSV_V = 0.2\n_C.INPUT.HSV_PROB = 0.9\n# RandomBrightnessContrast paramters\n_C.INPUT.BC_B = 0.2\n_C.INPUT.BC_C = 0.2\n_C.INPUT.BC_PROB = 0.9\n# Color paramters\n_C.INPUT.COLOR_PROB = 0.9\n# Random probability for ToGray\n_C.INPUT.TOFGRAY_PROB = 0.01\n# Random probability for HorizontalFlip\n_C.INPUT.HFLIP_PROB = 0.5\n# Random probability for VerticalFlip\n_C.INPUT.VFLIP_PROB = 0.5\n# Coutout paramters\n_C.INPUT.COTOUT_NUM_HOLES = 8\n_C.INPUT.COTOUT_MAX_H_SIZE = 64\n_C.INPUT.COTOUT_MAX_W_SIZE = 64\n_C.INPUT.COTOUT_FILL_VALUE = 0\n_C.INPUT.COTOUT_PROB = 0.5\n\n# -----------------------------------------------------------------------------\n# Dataset\n# -----------------------------------------------------------------------------\n_C.DATASETS = CN()\n# Root dir of dataset\n_C.DATASETS.ROOT_DIR = \"/content/global-wheat-detection\"\n_C.DATASETS.DIR = \"/content/global-wheat-detection\"\n# Fold to validate\n_C.DATASETS.VALID_FOLD = 0\n# # List of the dataset names for training, as present in paths_catalog.py\n# _C.DATASETS.TRAIN = ()\n# # List of the dataset names for testing, as present in paths_catalog.py\n# _C.DATASETS.TEST = ()\n\n# -----------------------------------------------------------------------------\n# DataLoader\n# -----------------------------------------------------------------------------\n_C.DATALOADER = CN()\n# Number of data loading threads\n_C.DATALOADER.NUM_WORKERS = 2\n\n# ---------------------------------------------------------------------------- #\n# Solver\n# ---------------------------------------------------------------------------- #\n_C.SOLVER = CN()\n_C.SOLVER.OPTIMIZER_NAME = \"SGD\"\n_C.SOLVER.SCHEDULER_NAME = \"CosineAnnealingWarmRestarts\"\n_C.SOLVER.COS_CPOCH = 2\n_C.SOLVER.T_MUL = 2\n\n_C.SOLVER.MAX_EPOCHS = 72\n\n_C.SOLVER.BASE_LR = 0.005\n_C.SOLVER.BIAS_LR_FACTOR = 1\n\n_C.SOLVER.MOMENTUM = 0.9\n\n_C.SOLVER.WEIGHT_DECAY = 0.0005\n_C.SOLVER.WEIGHT_DECAY_BIAS = 0\n_C.SOLVER.WEIGHT_DECAY_BN = 0\n\n_C.SOLVER.WARMUP_EPOCHS = 10\n\n_C.SOLVER.EARLY_STOP_PATIENCE = 20\n\n_C.SOLVER.TRAIN_CHECKPOINT = False\n_C.SOLVER.CLEAR_OUTPUT = True\n\n# Number of images per batch\n# This is global, so if we have 8 GPUs and IMS_PER_BATCH = 16, each GPU will\n# see 2 images per batch\n_C.SOLVER.IMS_PER_BATCH = 4\n\n# This is global, so if we have 8 GPUs and IMS_PER_BATCH = 16, each GPU will\n# see 2 images per batch\n_C.TEST = CN()\n_C.TEST.IMS_PER_BATCH = 4\n_C.TEST.WEIGHT = \"/content/output/best-checkpoint.bin\"\n\n# ---------------------------------------------------------------------------- #\n# Misc options\n# ---------------------------------------------------------------------------- #\n_C.OUTPUT_DIR = \"/content/drive/My Drive/Global_Wheat_Detection/experiments/baseline\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define TTA (Hide cell)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\n\nimport os\nimport warnings\nimport torch\nimport numpy as np\nfrom tqdm import tqdm\nimport pandas as pd\nfrom itertools import product\nimport sys\nsys.path.insert(0, \"./external/wbf\")\nimport ensemble_boxes\nwarnings.filterwarnings(\"ignore\")\n\n\nclass BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n\n    def batch_augment(self, images):\n        raise NotImplementedError\n\n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n\n    def batch_augment(self, images):\n        return images.flip(2)\n\n    def deaugment_boxes(self, boxes):\n        boxes[:, [1, 3]] = self.image_size - boxes[:, [3, 1]]\n        return boxes\n\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(2)\n\n    def batch_augment(self, images):\n        return images.flip(3)\n\n    def deaugment_boxes(self, boxes):\n        boxes[:, [0, 2]] = self.image_size - boxes[:, [2, 0]]\n        return boxes\n\n\nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n\n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0, 2]] = self.image_size - boxes[:, [3, 1]]\n        res_boxes[:, [1, 3]] = boxes[:, [0, 2]]\n        return res_boxes\n\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def __init__(self, transforms):\n        self.transforms = transforms\n\n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n\n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n\n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:, 0] = np.min(boxes[:, [0, 2]], axis=1)\n        result_boxes[:, 2] = np.max(boxes[:, [0, 2]], axis=1)\n        result_boxes[:, 1] = np.min(boxes[:, [1, 3]], axis=1)\n        result_boxes[:, 3] = np.max(boxes[:, [1, 3]], axis=1)\n        return result_boxes\n\n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from evaluate.evaluate import evaluate\n\nclass Tester:\n    def __init__(self, model, device, cfg, test_loader):\n        self.config = cfg\n        self.test_loader = test_loader\n\n        self.base_dir = f'{self.config.OUTPUT_DIR}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.score_threshold = SCORE_THRESHOLD\n        self.iou_threshold = NMS_IOU_THRESHOLD\n        self.use_nms = USE_NMS\n        \n        self.model = model\n        self.model.eval()\n\n        self.device = device\n        self.model.to(self.device)\n\n        self.log(f'Tester prepared. Device is {self.device}')\n\n    def test(self):\n        all_predictions = self.infer(pp_shrink=-1)\n        return all_predictions\n#         best_score_threshold, best_final_score = evaluate(all_predictions)\n#         return best_score_threshold, best_final_score, all_predictions\n#         self.save_predictions(results)\n\n    def process_det(self, index, outputs):\n        boxes = outputs[index]['boxes'].data.cpu().numpy()\n        scores = outputs[index]['scores'].data.cpu().numpy()\n        boxes = (boxes).clip(min=0, max=1023).astype(int)\n        indexes = np.where(scores > self.score_threshold)\n        boxes = boxes[indexes]\n        scores = scores[indexes]\n        return boxes, scores\n\n    def make_tta_predictions(self, tta_transforms, images):\n        with torch.no_grad():\n            images = torch.stack(images).float().cuda()\n            predictions = []\n            for tta_transform in tta_transforms:\n                result = []\n                outputs = self.model(tta_transform.batch_augment(images.clone()))\n                \n                \n                for i, image in enumerate(images):\n                    boxes = outputs[i]['boxes'].data.cpu().numpy()\n                    scores = outputs[i]['scores'].data.cpu().numpy()\n                    indexes = np.where(scores > self.score_threshold)[0]\n                    boxes = tta_transform.deaugment_boxes(boxes.copy())\n                    \n                    if self.use_nms: \n                        labels = np.ones(scores.shape[0]).astype(int).tolist()\n                        boxes, scores, labels = ensemble_boxes.ensemble_boxes_nms.nms_method([boxes], [scores], [labels], method=3,\n                                                                                        weights=None, iou_thr=self.iou_threshold,\n                                                                                        thresh=self.score_threshold)\n                    else: # not use NMS, just filter by confidence score\n                        boxes = boxes[indexes]\n                        scores = scores[indexes]\n                    result.append({\n                        'boxes': boxes,\n                        'scores': scores,\n                    })\n                predictions.append(result)\n        return predictions\n    \n    def run_wbf(self, predictions, image_index, image_size=IMG_SIZE, iou_thr=WBF_IOU, skip_box_thr=WBF_SKIP_BOX, weights=None):\n        boxes = [(prediction[image_index]['boxes'] / (image_size - 1)).tolist() for prediction in predictions]\n        scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n        labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in\n                  predictions]\n        boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels,\n                                                                                        weights=None, iou_thr=iou_thr,\n                                                                                        skip_box_thr=skip_box_thr)\n        boxes = boxes * (image_size - 1)\n        return boxes, scores, labels\n\n    def format_prediction_string(self, boxes, scores):\n        pred_strings = []\n        for j in zip(scores, boxes):\n            pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n        return \" \".join(pred_strings)\n\n    def infer(self, pp_shrink=-1):\n        self.model.eval()\n        torch.cuda.empty_cache()\n        \n        tta_transforms = []\n        for tta_combination in product([TTAHorizontalFlip(), None],\n#                                        [TTAVerticalFlip(), None],[TTARotate90(), None]\n                                      ):\n            tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))\n        test_loader = tqdm(self.test_loader, total=len(self.test_loader), desc=\"Testing\")\n        results = []\n        \n        boxes10 = []\n        \n        all_predictions = []\n        \n        for images, targets, image_ids in test_loader: # test_loader must contains targets in valid inference\n            predictions = self.make_tta_predictions(tta_transforms, images)\n            for i, image in enumerate(images):\n                boxes, scores, labels = self.run_wbf(predictions, image_index=i)\n                boxes = boxes.round().astype(np.int32).clip(min=0, max=1023)\n                image_id = image_ids[i]\n                \n                if len(boxes10) < 10:\n                    print('writing ... ',i,image_id)\n                    sample = image.permute(1,2,0).cpu().numpy()\n\n                    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n                    boxes10.append((sample, boxes))\n                    \n                    for box, score in zip(boxes,scores):\n                        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (0, 0, 1), 4)\n                        cv2.putText(sample, '%.2f'%(score), (box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX ,  \n                   1, (255,255,255), 3, cv2.LINE_AA)\n                    \n                    ax.set_axis_off()\n                    ax.imshow(sample);\n                    plt.show()\n                    \n                boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n                \n                #post-processing box shrinking as host advised boxes in test are tight to image\n                boxes[:, 0] = boxes[:, 0] + pp_shrink\n                boxes[:, 1] = boxes[:, 1] + pp_shrink\n                boxes[:, 2] = boxes[:, 2] - pp_shrink \n                boxes[:, 3] = boxes[:, 3] - pp_shrink \n                \n                #validation in case boxes are expanded not shrunk that they don't go beyond the boundaries of what is acceptable\n                for ppa in range(0,4):\n                    boxes[:, ppa] = [max(min(x, 1023), 0) for x in boxes[:, ppa]]\n                \n                gt = targets[i]['boxes'].cpu().numpy().astype(int)\n                gt[:, 2] = gt[:, 2] - gt[:, 0]\n                gt[:, 3] = gt[:, 3] - gt[:, 1]\n            \n                all_prediction = {\n                    'pred_boxes': boxes,\n                    'scores': scores,\n                    'gt_boxes': gt,\n                    'image_id': image_id,\n                }\n\n                all_predictions.append(all_prediction)\n                \n        return all_predictions\n\n    def format_prediction_string(self, boxes, scores):\n        pred_strings = []\n        for j in zip(scores, boxes):\n            pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n        return \" \".join(pred_strings)\n\n    def save_predictions(self, results):\n        test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\n        test_df.to_csv(f'{self.config.OUTPUT_DIR}/submission.csv', index=False)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n\n    def log(self, message):\n        if self.config.VERBOSE:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split fold and make correct CV with TTA/WBF/NMS\nSplit valid fold : specify fold in `cfg.DATASETS.VALID_FOLD`","metadata":{}},{"cell_type":"code","source":"from config import cfg\n\n#IMPORTANT\ncfg.DATASETS.VALID_FOLD = 0\n\ncfg['OUTPUT_DIR'] = \"/kaggle/working/\"\ncfg['DATASETS']['ROOT_DIR'] = \"/kaggle/input/global-wheat-detection\"\ncfg['DATASETS']['DIR'] = \"/kaggle/input/mydata/test.csv\"\ncfg['TEST']['WEIGHT'] = BEST_PATH\ncfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from data import make_data_loader\nval_loader = make_data_loader(cfg, is_train=False)\n# for x,y in enumerate(val_loader):\n#     print(x,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\n\nfrom os import mkdir\nsys.path.append('.')\nfrom config import cfg\nfrom data import make_test_data_loader\nfrom modeling import build_model\nfrom utils.logger import setup_logger\n\n# start here!!\nif True:\n    cfg.freeze()\n\n    output_dir = cfg.OUTPUT_DIR\n    if output_dir and not os.path.exists(output_dir):\n        print('creating ',cfg.OUTPUT_DIR)\n        mkdir(output_dir)\n\n    model = build_model(cfg)\n    device = cfg.MODEL.DEVICE\n    checkpoint = torch.load(cfg.TEST.WEIGHT)\n\n    tester = Tester(model=model, device=device, cfg=cfg, test_loader=val_loader)\n    tester.load(cfg['TEST']['WEIGHT'])\n    print('*** success load weights! ***')\n    \n    all_predictions = tester.test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numba\nfrom evaluate.calculate_score import *\ndef calculate_individual_scores(all_predictions, score_threshold):\n    final_scores = []\n    final_missed_boxes_nums = []\n    image_id_list = []\n    # Numba typed list!\n    iou_thresholds = numba.typed.List()\n\n    for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n        iou_thresholds.append(x)\n\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        pred_boxes = all_predictions[i]['pred_boxes'].copy()\n        scores = all_predictions[i]['scores'].copy()\n        image_id = all_predictions[i]['image_id']\n        \n        indexes = np.where(scores > score_threshold)\n        pred_boxes = pred_boxes[indexes]\n        scores = scores[indexes]\n        \n        image_precision = calculate_image_precision(gt_boxes, pred_boxes, thresholds=iou_thresholds, \n                                                    form='coco'#'pascal_voc'\n                                                   )\n        final_scores.append(image_precision)\n        image_id_list.append(image_id)\n    return final_scores, image_id_list, np.mean(final_scores)\n\ndef evaluate(all_predictions):\n    best_final_score, best_score_threshold = 0, 0\n    for score_threshold in tqdm(np.arange(0, 1, 0.05), total=np.arange(0, 1, 0.05).shape[0], desc=\"OOF\"):\n        _,_,final_score = calculate_individual_scores(all_predictions, score_threshold)\n        if final_score > best_final_score:\n            best_final_score = final_score\n            best_score_threshold = score_threshold\n\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        pred_boxes = all_predictions[i]['pred_boxes'].copy()\n        scores = all_predictions[i]['scores'].copy()\n        indexes = np.where(scores>best_score_threshold)\n        pred_boxes = pred_boxes[indexes]\n        all_predictions[i]['final_missed_boxes_nums'] = len(gt_boxes)-len(pred_boxes)\n\n    return best_score_threshold, best_final_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correct CV is printed out here!\nThis CV taking into account TTA and WBF","metadata":{}},{"cell_type":"code","source":"best_score_threshold, best_final_score = evaluate(all_predictions)\n\nprint('CV -- mAP .5:.75 for valid inference with best threshold = %.2f, TTA/WBF/NMS = %.4f' % (best_score_threshold, best_final_score))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### See the 20 worst predicted images\nNOte that normally we have good performance, these are just 'worst' images","metadata":{}}]}