{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Apache License 2.0 YoloV4 Pseudo Labeling + OOF Evaluation\n\nThanks everyone who shared about EfficientDet、Yolov4、Pseudo Labeling and OOF, let me finished this notebook.\n\nReferences:  \n\nLast original : https://www.kaggle.com/hawkey/yolov5-pseudo-labeling-oof-evaluation\n\nAwesome original Pseudo Labeling notebook : https://www.kaggle.com/nvnnghia/yolov5-pseudo-labeling  \nEvaluation Script : https://www.kaggle.com/pestipeti/competition-metric-details-script  \nOOF-Evaluation : https://www.kaggle.com/shonenkov/oof-evaluation-mixup-efficientdet  \nBayesian Optimization (though failed to improve my results) : https://www.kaggle.com/shonenkov/bayesian-optimization-wbf-efficientdet  \n\nYolov4 : https://www.kaggle.com/zacchaeus/train-yolov4-pytorch\n\nEfficientDet : https://www.kaggle.com/shonenkov/inference-efficientdet\n\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom tqdm.auto import tqdm\nimport shutil as sh\nimport torch\n#import sys\nfrom glob import glob\nimport time\nfrom pathlib import Path\nTIME_LIMIT = 6.9*60*60 # about 9 hours runtime limit\n#KERNEL_START_TIME = time.time()\n##Pesudo\nOrigin = False\np_HF = True\np_ROT = False\neimg_size = 1024\npesudo_thres = 0.4\npesudo_iou = 0.35\npesudo_score_threshold = 0\n###########################\nimg_size = 1024\nNMS_IOU_THR = 0.6\nNMS_CONF_THR = 0.35\n# WBF\nbest_iou_thr = 0.35\nbest_skip_box_thr = 0.35\n\n# Box conf threshold\nbest_final_score = 0\nbest_score_threshold = 0.1\n\nEPO = 8 #15-start #80-last\n\nWEIGHTS = \"../input/yolov4-pytorch/last_trainF5_onlyfix.weights\"\n\nPESUDO_WEIGHTS = '../input/effdet-cutmix-augmix-result/best-checkpoint_d7.bin'\n\nCONFIG = '../input/yolov4-pytorch/cd53s_self_1class.cfg'\n\nOPTIMIZER = '../input/yolov4-pytorch/last_optimizer.pth'\n\nis_TEST = len(os.listdir('../input/global-wheat-detection/test/'))>11\n\nis_AUG = False\nis_ROT = True\n\nVALIDATE = False\n\nPSEUDO = True\n####Predict\nYolov4 = True\nEffdet = True\nGET_PESUDO = True\n# For OOF evaluation\ncsv_file = '../input/yolov4-pytorch/newtrain.csv'\n#csv_file = '../input/global-wheat-detection/train.csv'\nmarking = pd.read_csv(csv_file)\n\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:,i]\nmarking.drop(columns=['bbox'], inplace=True)\nmax_label_numbers = 0\npesudo_result = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"####\n!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchTrain,DetBenchEval\nfrom effdet.efficientdet import HeadNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0, \"../input/pytorchyolov4/pytorch-YOLOv4\")\n#sys.path.insert(0, \"../input/newyolov4-pytorch/pytorch-YOLOv4\")\nfrom tool.darknet2pytorch import Darknet\nfrom tool.utils import *\nfrom tool.torch_utils import *\nimport argparse\nimport cv2\n!cp -r ../input/pytorchyolov4/pytorch-YOLOv4/* .\n#!cp -r ../input/newyolov4-pytorch/pytorch-YOLOv4/* .\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\ndef process_det(index, det, score_threshold=NMS_CONF_THR):\n    scores = det[index][:, 5].copy()\n    det = det[index][:, :4].copy()\n    bboxes = np.zeros((det.shape))\n    bboxes[:, 0] = ((det[:, 0] - det[:, 2] / 2) * 1024).astype(int)\n    bboxes[:, 1] = ((det[:, 1] - det[:, 3] / 2) * 1024).astype(int)\n    bboxes[:, 2] = ((det[:, 0] + det[:, 2] / 2) * 1024).astype(int)\n    bboxes[:, 3] = ((det[:, 1] + det[:, 3] / 2) * 1024).astype(int)\n    bboxes = (bboxes).clip(min = 0, max = 1024).astype(int)\n    \n    indexes = np.where(scores>score_threshold)\n    bboxes = bboxes[indexes]\n    scores = scores[indexes]\n    return bboxes, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convertTrainLabel(max_label_numbers):\n    df = pd.read_csv(csv_file)\n    bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        df[column] = bboxs[:,i]\n    df.drop(columns=['bbox'], inplace=True)\n    df['x1'] = df['x'] + df['w']\n    df['y1'] = df['y'] + df['h']\n    df['x_center'] = df['x'] + df['w']/2\n    df['y_center'] = df['y'] + df['h']/2\n    df['classes'] = 0\n    from tqdm.auto import tqdm\n    import shutil as sh\n    df = df[['image_id','x', 'y', 'w', 'h','x1','y1','x_center','y_center','classes']]\n    \n    index = list(set(df.image_id))\n    source = 'train'\n    if True:\n        for fold in [0]:\n            val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]\n            for name,mini in tqdm(df.groupby('image_id')):\n                os.makedirs('convertor', exist_ok=True)\n                sh.copy(\"../input/global-wheat-detection/{}/{}.jpg\".format(source,name),'convertor/{}.jpg'.format(name))\n                if name in val_index:\n                    path2save = 'convertor/val.txt'\n                else:\n                    path2save = 'convertor/train.txt'\n                with open(path2save, 'a') as f:\n                    f.write(f'{name}.jpg')\n                    row = mini[['x','y','x1','y1','classes']].astype(int).values\n                    if row.shape[0]>max_label_numbers:\n                        max_label_numbers = row.shape[0]\n                    # row = row/1024\n                    row = row.astype(str)\n                    for j in range(len(row)):\n                        text = ' '+','.join(row[j])\n                        f.write(text)\n                    f.write('\\n')\n    return max_label_numbers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ensemble_boxes import *\n\ndef run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):\n    labels = [np.zeros(score.shape[0]) for score in scores]\n    boxes = [box/(image_size) for box in boxes]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size)\n    return boxes, scores, labels\n\ndef TTAImage(image, index):\n    image1 = image.copy()\n    if index==0: \n        rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image\n    elif index==1:\n        rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image2\n    elif index==2:\n        rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image3\n    elif index == 3:\n        return image1\n    \ndef rotBoxes90(boxes, im_w, im_h):\n    ret_boxes =[]\n    for box in boxes:\n        x1, y1, x2, y2 = box\n        x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2\n        x1, y1, x2, y2 = y1, -x1, y2, -x2\n        x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)\n        x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)\n        ret_boxes.append([x1a, y1a, x2a, y2a])\n    return np.array(ret_boxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# validate\n\nimport pandas as pd\nimport numpy as np\nimport numba\nimport re\nimport cv2\nimport ast\nimport matplotlib.pyplot as plt\n\nfrom numba import jit\nfrom typing import List, Union, Tuple\n\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision\n\n    \n# Numba typed list!\niou_thresholds = numba.typed.List()\n\nfor x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:\n    iou_thresholds.append(x)\ndef validate():\n    \"\"\"hyper parameters\"\"\"\n    use_cuda = True\n    truth = {}\n    DATA_ROOT_PATH = '../input/global-wheat-detection/train'\n    with open('convertor/val.txt', 'r', encoding='utf-8') as f:\n        for line in f.readlines():\n            data = line.split(\" \")\n            truth[data[0]] = []\n            for i in data[1:]:\n                truth[data[0]].append([int(j) for j in i.split(',')])\n\n    image_ids = [image_id.split('.')[0] for image_id in list(truth.keys())]\n    #image_ids = image_ids[0:10]\n    \n    weights = 'checkpoints/last.pt'\n    Effdet = False\n    if not os.path.exists(weights):\n        Effdet = True\n        model = Darknet(CONFIG)\n        model.load_weights(WEIGHTS)\n        print('OLD WEIGHT')\n    else:\n        model = torch.load(weights)\n\n    if use_cuda:\n        model.cuda()\n        \n    if Effdet:\n        net = get_valnet_file(PESUDO_WEIGHTS)\n        \n    model.eval()\n    num_classes = model.num_classes\n    class_names = 'wheat'\n    \n    results = []\n    \n    for image_id in image_ids:\n        enboxes = []\n        enscores = []\n        \n        img = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg')\n        img = cv2.resize(img, (model.width, model.height))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        if Yolov4:\n            if is_ROT:\n                for i in range(4):\n                    img1 = TTAImage(img, i)\n                    img1 = img1.transpose(2, 0, 1)      \n                    img1 = torch.from_numpy(img1).float().div(255.0).unsqueeze(0)\n                    img1 = img1.cuda()\n                    img1 = torch.autograd.Variable(img1)\n                    with torch.no_grad():\n                        output = model(img1)\n                        boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)\n                        det = np.array(boxes)\n                        boxes, scores = process_det(0, det, NMS_CONF_THR)\n                        for _ in range(3-i):\n                            boxes = rotBoxes90(boxes, *img.shape[:2])  \n                        enboxes.append(boxes)\n                        enscores.append(scores) \n            with torch.no_grad():\n                img1 = img.transpose(2, 0, 1)      \n                img1 = torch.from_numpy(img1).float().div(255.0).unsqueeze(0)\n                img1 = img1.cuda()\n                img1 = torch.autograd.Variable(img1)\n                output = model(img1)\n                boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)\n                det = np.array(boxes)\n                boxes, scores = process_det(0, det, NMS_CONF_THR)\n                enboxes.append(boxes)\n                enscores.append(scores) \n            \n        if Effdet:\n            img1 = img.transpose(2, 0, 1)\n            img1 = torch.from_numpy(img1).cuda()\n            img1 =  img1.float()  # uint8 to fp16/32\n            img1 /= 255.0\n            if p_HF:\n                with torch.no_grad():\n                    imgs = torch.stack((img1, )).float().cuda()\n                    for tta_transform in tta_transforms:\n                        det = net(tta_transform.batch_augment(imgs.clone()), torch.tensor([1]*imgs.shape[0]).float().cuda())\n\n                        for i in range(imgs.shape[0]):\n                            boxes = det[i].detach().cpu().numpy()[:,:4]    \n                            scores = det[i].detach().cpu().numpy()[:,4]\n                            indexes = np.where(scores > pesudo_thres)[0]\n                            boxes = boxes[indexes]\n                            boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                            boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                            boxes = tta_transform.deaugment_boxes(boxes.copy())\n                            enboxes.append(boxes)\n                            enscores.append(scores) \n   \n        records = marking[marking['image_id'] == image_id]\n        gtboxes = records[['x', 'y', 'w', 'h']].values\n        gtboxes = gtboxes.astype(np.int32).clip(min=0, max=1024)\n        gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]\n        gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]\n            \n        result = {\n            'image_id': image_id,\n            'pred_enboxes': enboxes, # xyx2y2\n            'pred_enscores': enscores,\n            'gt_boxes': gtboxes, # xyx2y2\n        }\n        results.append(result)\n        \n    return results\n\ndef calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):\n    final_scores = []\n    for i in range(len(all_predictions)):\n        gt_boxes = all_predictions[i]['gt_boxes'].copy()\n        enboxes = all_predictions[i]['pred_enboxes'].copy()\n        enscores = all_predictions[i]['pred_enscores'].copy()\n        image_id = all_predictions[i]['image_id']\n        \n        pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)    \n        pred_boxes = pred_boxes.astype(np.int32).clip(min=0, max=1024)\n\n        indexes = np.where(scores>score_threshold)\n        pred_boxes = pred_boxes[indexes]\n        scores = scores[indexes]\n\n        image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')\n        final_scores.append(image_precision)\n\n    return np.mean(final_scores)\n\ndef show_result(sample_id, preds, gt_boxes):\n    sample = cv2.imread(f'../input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)\n    sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for pred_box in preds:\n        cv2.rectangle(\n            sample,\n            (pred_box[0], pred_box[1]),\n            (pred_box[2], pred_box[3]),\n            (220, 0, 0), 2\n        )\n\n    for gt_box in gt_boxes:    \n        cv2.rectangle(\n            sample,\n            (gt_box[0], gt_box[1]),\n            (gt_box[2], gt_box[3]),\n            (0, 0, 220), 2\n        )\n\n    ax.set_axis_off()\n    ax.imshow(sample)\n    ax.set_title(\"RED: Predicted | BLUE - Ground-truth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bayesian Optimize\n\nfrom skopt import gp_minimize, forest_minimize\nfrom skopt.utils import use_named_args\nfrom skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret\nfrom skopt.space import Categorical, Integer, Real\n\ndef log(text):\n    print(text)\n\ndef optimize(space, all_predictions, n_calls=10):\n    @use_named_args(space)\n    def score(**params):\n        \n        log('-'*10)\n        log(params)\n        final_score = calculate_final_score(all_predictions, **params)\n        log(f'final_score = {final_score}')\n        log('-'*10)\n        return -final_score\n\n    return gp_minimize(func=score, dimensions=space, n_calls=n_calls)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EFFD_PESUDO","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom torch.utils.data import Dataset, DataLoader\nimport path\nimg_size = 1024\ndef get_valid_transforms():\n    return A.Compose([\n        A.Resize(height=img_size, width=img_size, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.0)\n\nDATA_ROOT_PATH = '../input/global-wheat-detection/test'\n\nclass DatasetRetriever_test(Dataset):\n\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        im0 = image\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        return image, image_id, im0\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\ndataset = DatasetRetriever_test(\n    image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),\n    transforms=get_valid_transforms()\n)\ndata_loader = DataLoader(\n        dataset,\n        batch_size=2,\n        shuffle=False,\n        num_workers=4,\n        drop_last=False\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import product\nclass BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = img_size\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n\ntta_transforms = []\nfor tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n    tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_struct = 'tf_efficientdet_d7'\ndef get_valnet_file(Get_Path):\n    config = get_efficientdet_config(model_struct)\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size = img_size\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(Get_Path)\n    net.load_state_dict(checkpoint['model_state_dict'])\n\n    del checkpoint\n    #gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\ndef makePseudolabel():\n    \"\"\"hyper parameters\"\"\"\n    use_cuda = True\n    max_label_numbers = 0\n    DATA_ROOT_PATH = '../input/global-wheat-detection/test'\n    image_ids = [path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]\n    \n    model = Darknet(CONFIG)\n    model.load_weights(WEIGHTS)\n\n    if use_cuda:\n        model.cuda()\n        \n    if Effdet:\n        net = get_valnet_file(PESUDO_WEIGHTS)\n        \n    model.eval()\n    num_classes = model.num_classes\n    class_names = 'wheat'\n    \n    count = 0\n    pesudo_result = []\n    for image_id in image_ids:\n        enboxes = []\n        enscores = []\n        try:\n            img = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg')\n            img = cv2.resize(img, (model.width, model.height))\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            if Yolov4:\n                if is_ROT:\n                    for i in range(4):\n                        img1 = TTAImage(img, i)\n                        img1 = img1.transpose(2, 0, 1)      \n                        img1 = torch.from_numpy(img1).float().div(255.0).unsqueeze(0)\n                        img1 = img1.cuda()\n                        img1 = torch.autograd.Variable(img1)\n                        with torch.no_grad():\n                            output = model(img1)\n                            boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)\n                            det = np.array(boxes)\n                            if det.size !=0:\n                                boxes, scores = process_det(0, det, NMS_CONF_THR)\n                                for _ in range(3-i):\n                                    boxes = rotBoxes90(boxes, *img.shape[:2])  \n                                enboxes.append(boxes)\n                                enscores.append(scores) \n                with torch.no_grad():\n                    img1 = img.transpose(2, 0, 1)      \n                    img1 = torch.from_numpy(img1).float().div(255.0).unsqueeze(0)\n                    img1 = img1.cuda()\n                    img1 = torch.autograd.Variable(img1)\n                    output = model(img1)\n                    boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)\n                    det = np.array(boxes)\n                    if det.size !=0:\n                        boxes, scores = process_det(0, det, NMS_CONF_THR)\n                        enboxes.append(boxes)\n                        enscores.append(scores) \n\n            if Effdet:\n                img1 = img.transpose(2, 0, 1)\n                img1 = torch.from_numpy(img1).cuda()\n                img1 =  img1.float()  # uint8 to fp16/32\n                img1 /= 255.0\n                if p_HF:\n                    with torch.no_grad():\n                        imgs = torch.stack((img1, )).float().cuda()\n                        for tta_transform in tta_transforms:\n                            det = net(tta_transform.batch_augment(imgs.clone()), torch.tensor([1]*imgs.shape[0]).float().cuda())\n\n                            for i in range(imgs.shape[0]):\n                                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                                scores = det[i].detach().cpu().numpy()[:,4]\n                                indexes = np.where(scores > pesudo_thres)[0]\n                                boxes = boxes[indexes]\n                                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                                boxes = tta_transform.deaugment_boxes(boxes.copy())\n                                enboxes.append(boxes)\n                                enscores.append(scores) \n\n            boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)    \n            boxes = boxes.astype(np.int32).clip(min=0, max=1024)\n            #boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n            #boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n            indices = scores >= best_score_threshold\n            boxes = boxes[indices]\n            scores = scores[indices]\n            lineo = ''\n            for box in boxes:\n                x, y, x1, y1 = box\n                lineo += ' %d,%d,%d,%d,0'%(x, y, x1, y1)\n            os.makedirs('convertor', exist_ok=True)\n            if len(lineo)>0:\n                with open('convertor/train.txt', 'a') as f:\n                    f.write(f'{image_id}.jpg')\n                    f.write(lineo)\n                    f.write('\\n')\n                sh.copy(\"../input/global-wheat-detection/test/{}.jpg\".format(image_id),'convertor/{}.jpg'.format(image_id))\n                result = {\n                    'image_id': image_id,\n                    'box': enboxes,\n                    'score': enscores\n                } \n                if boxes.shape[0]>max_label_numbers:\n                    max_label_numbers = boxes.shape[0]\n                pesudo_result.append(result)\n                print(path,image_id)\n        except:\n            boxes = []\n            scores = []\n            print('one error')\n        \n    return pesudo_result, max_label_numbers\n    #q.put(pesudo_result)\n    #.put(max_label_numbers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if PSEUDO or is_TEST:\n    max_label_numbers = convertTrainLabel(max_label_numbers)\nif is_TEST and PSEUDO:\n    \"\"\"\n    import multiprocessing\n    q = multiprocessing.Queue()\n    p = multiprocessing.Process(target=makePseudolabel, args=(q,))\n    p.start()\n    p.join()\n    pesudo_result = q.get()\n    pesudo_label_numbers = q.get()\n    #\n     \"\"\"  \n    pesudo_result, pesudo_label_numbers = makePseudolabel()\n    if pesudo_label_numbers > max_label_numbers:\n        max_label_numbers = pesudo_label_numbers\nelse:\n    pesudo_result = []\n#os.listdir('convertor/')\n#os.remove('convertor/train.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_TEST:\n    if PSEUDO:\n        #remain_times = int( TIME_LIMIT - (time.time() - KERNEL_START_TIME) )0\n        remain_times = 60\n        !python train.py -TRAIN_EPOCHS {EPO} -cfgfile {CONFIG} -op {OPTIMIZER} -dir /kaggle/working/convertor -pretrained {WEIGHTS} -train_label_path convertor/train.txt -val_label_path convertor/val.txt -optimizer radam -iou-type ciou -l 0.0001 -g 0 -classes 1 -maxboxes {max_label_numbers+10} -time_limit {remain_times} \n    else:\n        pass\nelse:\n    pass\n    #!python train.py --weights {WEIGHTS} --img 1024 --batch 2 --epochs 1 --data {DATA} --cfg {CONFIG}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if VALIDATE and is_TEST:\n    all_predictions = validate()\n    \n    # Bayesian Optimization: worse results.\n    '''\n    space = [\n        Real(0, 1, name='iou_thr'),\n        Real(0.25, 1, name='skip_box_thr'),\n        Real(0, 1, name='score_threshold'),\n    ]\n\n    opt_result = optimize(\n        space, \n        all_predictions,\n        n_calls=50,\n    )\n    \n    best_final_score = opt_result.fun\n    best_iou_thr = opt_result.x[0]\n    best_skip_box_thr = opt_result.x[1]\n    best_score_threshold = opt_result.x[2]\n\n\n    print('-'*13 + 'WBF' + '-'*14)\n    print(\"[Baseline score]\", calculate_final_score(all_predictions, 0.6, 0.43, 0))\n    print(f'[Best Iou Thr]: {best_iou_thr:.3f}')\n    print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')\n    print(f'[Best Score Thr]: {best_score_threshold:.3f}')\n    print(f'[Best Score]: {best_final_score:.4f}')\n    print('-'*30)\n    \n    '''\n    \n    for score_threshold in tqdm(np.arange(0.1, 0.25, 0.01), total=np.arange(0.1, 0.25, 0.01).shape[0]):\n        final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)\n        if final_score > best_final_score:\n            best_final_score = final_score\n            best_score_threshold = score_threshold\n\n    print('-'*30)\n    print(f'[Best Score Threshold]: {best_score_threshold}')\n    print(f'[OOF Score]: {best_final_score:.4f}')\n    print('-'*30)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)\n\ndef detect_cv2(pesudo_result):\n    \"\"\"hyper parameters\"\"\"\n    use_cuda = True\n    DATA_ROOT_PATH = '../input/global-wheat-detection/test'\n    image_ids = [path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]\n    \n    weights = 'checkpoints/last.pt'\n    Effdet = False\n    if not os.path.exists(weights):\n        Effdet = True\n        model = Darknet(CONFIG)\n        model.load_weights(WEIGHTS)\n        print('OLD WEIGHT')\n    else:\n        #ckpt = torch.load(weights)\n        #model = Darknet(CONFIG)\n        #model.load_state_dict(ckpt)\n        model = torch.load(weights)\n    if (len(pesudo_result)>0) and (GET_PESUDO==True):\n        print('get pesudo label')\n        \n    if use_cuda:\n        model.cuda()\n        \n    if Effdet:\n        net = get_valnet_file(PESUDO_WEIGHTS)\n        \n    model.eval()\n    num_classes = model.num_classes\n    class_names = 'wheat'\n    \n    results = []\n    fig, ax = plt.subplots(5, 2, figsize=(30, 70))\n    count = 0\n\n    for image_id in image_ids:\n        enboxes = []\n        enscores = []\n        try:\n            img = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg')\n            h, w = img.shape[:2]\n            img = cv2.resize(img, (model.width, model.height))\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            if Yolov4:\n                if is_ROT:\n                    for i in range(4):\n                        img1 = TTAImage(img, i)\n                        with torch.no_grad():\n                            img1 = img1.transpose(2, 0, 1)      \n                            img1 = torch.from_numpy(img1).float().unsqueeze(0).cuda()\n                            img1 /= 255.0\n                            img1 = torch.autograd.Variable(img1)\n                            output = model(img1)\n                            boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)\n                            det = np.array(boxes)\n                            boxes, scores = process_det(0, det, NMS_CONF_THR)\n                            for _ in range(3-i):\n                                boxes = rotBoxes90(boxes, *img.shape[:2])  \n                            enboxes.append(boxes)\n                            enscores.append(scores) \n                with torch.no_grad():\n                    img1 = img.transpose(2, 0, 1)      \n                    img1 = torch.from_numpy(img1).float().unsqueeze(0).cuda()\n                    img1 /= 255.0\n                    img1 = torch.autograd.Variable(img1)\n                    output = model(img1)\n                    boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)\n                    det = np.array(boxes)\n                    boxes, scores = process_det(0, det, NMS_CONF_THR)\n                    enboxes.append(boxes)\n                    enscores.append(scores) \n\n\n            if Effdet:\n                img1 = img.transpose(2, 0, 1)\n                img1 = torch.from_numpy(img1).cuda()\n                img1 =  img1.float()  # uint8 to fp16/32\n                img1 /= 255.0\n                if p_HF:\n                    with torch.no_grad():\n                        imgs = torch.stack((img1, )).float().cuda()\n                        for tta_transform in tta_transforms:\n                            det = net(tta_transform.batch_augment(imgs.clone()), torch.tensor([1]*imgs.shape[0]).float().cuda())\n\n                            for i in range(imgs.shape[0]):\n                                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                                scores = det[i].detach().cpu().numpy()[:,4]\n                                indexes = np.where(scores > pesudo_thres)[0]\n                                boxes = boxes[indexes]\n                                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                                boxes = tta_transform.deaugment_boxes(boxes.copy())\n                                enboxes.append(boxes)\n                                enscores.append(scores)\n\n            if (len(pesudo_result)>0) and (GET_PESUDO==True):  \n                if next((inx  for (inx,dic) in enumerate(pesudo_result) if dic['image_id']==image_id), None) is not None:\n                    enboxes.extend(pesudo_result[next((inx  for (inx,dic) in enumerate(pesudo_result) if dic['image_id']==image_id), None)]['box'])\n                    enscores.extend(pesudo_result[next((inx  for (inx,dic) in enumerate(pesudo_result) if dic['image_id']==image_id), None)]['score']) \n            boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)    \n            boxes = (boxes/1024*h).astype(np.int32).clip(min=0, max=h)  \n            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n            indices = scores >= best_score_threshold\n            boxes = boxes[indices]\n            scores = scores[indices]\n\n            if (count<10) and (is_TEST==False):\n                img_ = img  # BGR\n                for box, score in zip(boxes,scores):\n                    cv2.rectangle(img_, (box[0], box[1]), (box[2]+box[0], box[3]+box[1]), (220, 0, 0), 2)\n                    cv2.putText(img_, '%.2f'%(score), (box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,255,255), 2, cv2.LINE_AA)\n                ax[count%5][count//5].imshow(img_)\n                count+=1\n        except:\n            boxes = []\n            scores = []\n            print('one error')\n            pass\n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        } \n        results.append(result)\n    return results ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = detect_cv2(pesudo_result)\ntest_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf convertor","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}