{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About\n\nThis is a notebook designed to follow up on [this post](https://www.kaggle.com/c/global-wheat-detection/discussion/172827).\n\n### Who this is for\n- You participated in the [Global Wheat Detection](https://www.kaggle.com/c/global-wheat-detection) competition\n- It helps if you've worked with [Alex Shonenkov](https://www.kaggle.com/shonenkov)'s EfficientDet implementation. This notebook is a fork (of a fork of a fork :P) of [Alex's one](https://www.kaggle.com/shonenkov/inference-efficientdet).\n\nOtherwise, you can just read about the general idea [here](https://www.kaggle.com/c/global-wheat-detection/discussion/172827).\n\n### How to read\nIf you can tick off both points above and want to be able to get the value out of this notebook really fast, just follow my markdown prompts.\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Below I introduce `QUAD_SIZE` which determines the size of the quadrants I'll be analysing. I found 640 to be the Goldilocks number (not too small, not too large).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\n\nfrom ensemble_boxes import *\nimport torch\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom torch.utils.data import Dataset,DataLoader\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport cv2\nimport gc\nfrom matplotlib import pyplot as plt\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval\nfrom effdet.efficientdet import HeadNet\nfrom scipy.spatial.distance import cosine\nimport os\n\nFULL_SIZE = 1024\nINF_SIZE = 640\nQUAD_SIZE = 640\n\n# wbf params\nSKIP_BOX_THR = 0.43 # 0.43\nIOU_THR = 0.44 # 0.44","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations\n\nNothing new here if you've already done TTA.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### REVERSE AUGMENTATIONS FOR BBOXES\n\ndef hflip_predictions(predictions):\n    \"\"\"\n    horizontal flip\n    \"\"\"\n    for batch_ix in range(len(predictions[0])):\n        boxes = predictions[0][batch_ix]['boxes']\n        boxes[:,0], boxes[:,2] = INF_SIZE - boxes[:,2], INF_SIZE - boxes[:,0]\n        predictions[0][batch_ix]['boxes'] = boxes\n    return predictions\n\ndef vflip_predictions(predictions):\n    \"\"\"\n    vertical flip\n    \"\"\"\n    for batch_ix in range(len(predictions[0])):\n        boxes = predictions[0][batch_ix]['boxes']\n        boxes[:,1], boxes[:,3] = INF_SIZE - boxes[:,3], INF_SIZE - boxes[:,1]\n        predictions[0][batch_ix]['boxes'] = boxes\n    return predictions\n\ndef rotate_predictions(predictions):\n    \"\"\"\n    rotate counter-clockwise 90 degrees\n    \"\"\"\n    for batch_ix in range(len(predictions[0])):\n        boxes = predictions[0][batch_ix]['boxes']\n        x1 = boxes[:,0].copy()\n        x2 = boxes[:,2].copy()\n        y1 = boxes[:,1].copy()\n        y2 = boxes[:,3].copy()\n        h = (y2 - y1).copy()\n        boxes[:,0] = INF_SIZE - y1 - h\n        boxes[:,1] = x1\n        boxes[:,2] = INF_SIZE - y2 + h\n        boxes[:,3] = x2\n        predictions[0][batch_ix]['boxes'] = boxes\n    return predictions\n\n# i = do nothing (should only be on its own)\n# h = horizontal flip\n# v = vertical flip\n# r = rotation\n# applied from left to right\nAUG_TRANSFORMS = {\n    's': A.Resize(height=INF_SIZE, width=INF_SIZE, p=1.0),\n    't': ToTensorV2(p=1.0),\n    'h': A.HorizontalFlip(p=1.0),\n    'v': A.VerticalFlip(p=1.0),\n    'r': A.Rotate((90,90), p=1.0),\n    'c': A.OneOf([\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,\n                                     val_shift_limit=0.2, p=1),\n                A.RandomBrightnessContrast(brightness_limit=0.2, \n                                           contrast_limit=0.2, p=1),\n            ],p=1),\n}\n\nUNWRAP_FUNCS = {\n    'h': hflip_predictions,\n    'v': vflip_predictions,\n    'r': rotate_predictions,\n}\n\n# non-trivial augs\n# AUGS = ['', 'h', 'v', 'r', 'hv', 'rh', 'rv', 'rhv']\n# AUGS = ['', 'h', 'v', 'r']\nAUGS = ['', 'h', 'v', 'r']\n\ndef apply_augs(image):\n    sample = {'image': image}\n    images = []\n    for aug_str in AUGS:\n        # compose aug\n        transform_list = [AUG_TRANSFORMS['s']]\n        for aug_char in aug_str:\n            transform_list += [AUG_TRANSFORMS[aug_char]]\n        transform_list += [AUG_TRANSFORMS['t']]\n        transforms = A.Compose(transform_list, p=1.0)\n        # apply transforms\n        sample_aug = transforms(**sample)\n        images.append(sample_aug['image'])\n    return images\n\n\ndef unwrap_augs(ls_predictions):\n    \"\"\"\n    expects list of predictions as long as the AUGS list\n    one set of predictions for each aug\n    \"\"\"\n    ls_corrected_predictions = []\n    assert len(ls_predictions) == len(AUGS), \\\n        \"Warning length of predictions needs to be the same as length of AUGS\"\n    for aug_ix, aug_str in enumerate(AUGS):\n        corrected_predictions = ls_predictions[aug_ix]\n        # pass trivial aug without correction\n        if aug_str in ['', 'c']:\n            ls_corrected_predictions.append(corrected_predictions)\n            continue\n        # unwrap backwards\n        for aug_char in aug_str[::-1]:\n            corrected_predictions = UNWRAP_FUNCS[aug_char](corrected_predictions)    \n        ls_corrected_predictions.append(corrected_predictions)\n    return ls_corrected_predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data\n\nDefinitely some new stuff here. Look at how I cut out quadrants in the `__getitem__` function.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT_PATH = '../input/global-wheat-detection/test'\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, crop_size=QUAD_SIZE, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        do_stitch_edges = []\n        orig_image = self.load_image(image_id)\n        orig_image = cv2.resize(orig_image, (FULL_SIZE, FULL_SIZE))\n        # produce 4 crops\n        # 0th dim will be for quadrants and full image\n        quad_images = []\n        quad_images.append(orig_image[:QUAD_SIZE,:QUAD_SIZE,:]) # top left\n        quad_images.append(orig_image[:QUAD_SIZE,-QUAD_SIZE:,:]) # top right\n        quad_images.append(orig_image[-QUAD_SIZE:,:QUAD_SIZE,:]) # bottom left\n        quad_images.append(orig_image[-QUAD_SIZE:,-QUAD_SIZE:,:]) # bottom right\n        quad_images += [orig_image]\n        # 1st dim will be for augs\n        ls_images = []\n        for quad_image in quad_images:\n            aug_images = apply_augs(quad_image)\n            if self.transforms:\n                for i, aug_image in enumerate(aug_images):\n                    sample = {'image': aug_image}\n                    sample = self.transforms(**sample)\n                    aug_images[i] = sample['image']   \n            ls_images.append(aug_images)\n        return ls_images, image_id, do_stitch_edges\n    \n    def load_image(self, image_id):\n        image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        return image\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = DatasetRetriever(\n    image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]))\n\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndata_loader = DataLoader(\n    dataset,\n    batch_size=2,\n    shuffle=False,\n    num_workers=4,\n    drop_last=False,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Nets\n\nNothing new here.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_net(checkpoint_path):\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = 1\n    config.image_size=INF_SIZE\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint['model_state_dict'])\n\n    del checkpoint\n    gc.collect()\n\n    net = DetBenchEval(net, config)\n    net.eval();\n    return net.cuda()\n\nNETS = [\n    load_net('../input/alex-gwd-demo-models/fold1-84_(1)024-last.bin')\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helpers\n\nNew stuff here. `aug_ensemble_predictions` and `quad_ensemble_predictions` are designed to work together in a nested loop. We have 4 image orientations, and 5 images (4 quadrants + full image), so we are ensembling over 20 predictions per image.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_predictions(images, nets, score_threshold=0.22):\n    images = torch.stack(images).cuda().float()\n    predictions = []\n    with torch.no_grad():\n        dets = []\n        for net in nets:\n            dets.append(net(images, torch.tensor([1]*images.shape[0]).float().cuda()))\n        for i in range(images.shape[0]):\n            ls_boxes = []\n            ls_scores = []\n            for det in dets:\n                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                scores = det[i].detach().cpu().numpy()[:,4]\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                ls_boxes.append(boxes)\n                ls_scores.append(scores[indexes])\n            predictions.append({\n                'boxes': np.concatenate(ls_boxes),\n                'scores': np.concatenate(ls_scores),\n            })\n    return [predictions]\n\n\ndef run_wbf(predictions, image_index, image_size=INF_SIZE, weights=None):\n    boxes = [(prediction[image_index]['boxes']/image_size).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=IOU_THR, skip_box_thr=SKIP_BOX_THR)\n    boxes = boxes*image_size\n    return boxes, scores, labels\n\ndef prune_edge_predictions(predictions, quad_ix, size):\n    \"\"\"\n    get rid of any predictions touching the edges\n    or within buffer range\n    MUST BE DONE BEFORE scale_predictions\n    \"\"\"\n    boxes = predictions['boxes']\n    scores = predictions['scores']\n    buffer = 20\n    if len(boxes) > 0:\n        filt_left = boxes[:, 0] > (0 + buffer)\n        filt_top = boxes[:, 1] > (0 + buffer)\n        filt_right = boxes[:, 2] < (size - buffer)\n        filt_bottom = boxes[:, 3] < (size - buffer)\n        if quad_ix == 0: # top left\n            filt = np.bitwise_and(filt_right, filt_bottom)\n        elif quad_ix == 1: # top right\n            filt = np.bitwise_and(filt_left, filt_bottom)\n        elif quad_ix == 2: # bottom left\n            filt = np.bitwise_and(filt_right, filt_top)\n        elif quad_ix == 3: # bottom right\n            filt = np.bitwise_and(filt_left, filt_top)\n        boxes = boxes[filt]\n        scores = scores[filt]\n    predictions['boxes'] = boxes\n    predictions['scores'] = scores\n    return predictions\n\ndef scale_predictions(predictions, scaling):\n    predictions['boxes'] = predictions['boxes'] * scaling\n    return predictions\n\ndef shift_predictions(predictions, quad_ix, shift):\n    \"\"\"\n    0 ==> top left\n    1 ==> top right\n    2 ==> bottom left\n    3 ==> bottom right\n    MUST BE DONE BEFORE combine_predictions\n    \"\"\"\n    boxes = predictions['boxes']\n    boxes[:, 0] += shift * (quad_ix % 2)\n    boxes[:, 1] += shift * (0 if quad_ix < 2 else 1)\n    boxes[:, 2] += shift * (quad_ix % 2)\n    boxes[:, 3] += shift * (0 if quad_ix < 2 else 1)\n    predictions['boxes'] = boxes\n    return predictions\n\ndef combine_predictions(ls_predictions):\n    combined_predictions = []\n    # loop over batch dimension\n    for batch_ix in range(len(ls_predictions[0])):\n        boxes = []\n        scores = []\n        # loop over predictions sets\n        for pred_ix in range(len(ls_predictions)):\n            boxes.append(ls_predictions[pred_ix][batch_ix]['boxes'])\n            scores.append(ls_predictions[pred_ix][batch_ix]['scores'])\n        boxes = np.concatenate(boxes)\n        scores = np.concatenate(scores)\n        # sort from highest to lowest scrore\n        sort_ix = np.argsort(scores)[::-1]\n        boxes = np.array(boxes)[sort_ix]\n        scores = np.array(scores)[sort_ix]\n        combined_predictions.append({\n            'boxes': boxes,\n            'scores': scores\n        })\n    return [combined_predictions]\n\n\ndef aug_ensemble_predictions(aug_images, nets):\n    \"\"\"\n    aug_images is expected to be a list of n_augs augmentations each\n    with batch dim number of images\n    this will return aug_predictions which will have dimensionality (n_augs, n_batches)\n    \"\"\"\n    aug_predictions = []\n    for batch_images in aug_images:\n        aug_predictions.append(make_predictions(batch_images, nets))\n    aug_predictions = unwrap_augs(aug_predictions)\n    return aug_predictions\n\n\ndef quad_ensemble_predictions(ls_images, nets):\n    \"\"\"\n    ls_images is expected to be a 2D list with 5 images in the 1st dim,\n    first 4 for the quadrants, and last one for the full image\n    and n augmentations in the second dim\n    \"\"\"\n    ls_predictions = []\n    for quad_ix, aug_images in enumerate(ls_images):\n        aug_predictions = aug_ensemble_predictions(aug_images, nets)\n        for aug_ix, batched_predictions in enumerate(aug_predictions):\n            for batch_ix, predictions in enumerate(batched_predictions[0]):\n                if quad_ix < 4:\n                    batched_predictions[0][batch_ix] = prune_edge_predictions(predictions, quad_ix, INF_SIZE)\n                    batched_predictions[0][batch_ix] = scale_predictions(batched_predictions[0][batch_ix], QUAD_SIZE/INF_SIZE)\n                    batched_predictions[0][batch_ix] = shift_predictions(batched_predictions[0][batch_ix], quad_ix, FULL_SIZE-QUAD_SIZE)\n                else:\n                    batched_predictions[0][batch_ix] = scale_predictions(predictions, FULL_SIZE/INF_SIZE)\n            ls_predictions += batched_predictions\n    return ls_predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check if it works by predicting on a sample test image. Notice the reconstructions of the image from the quadrants. I could have used the original image, but it was useful to check that I was reassembling everything properly.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfor j, (images_batches, image_ids, do_stitch_edges) in enumerate(data_loader):\n    break\n\n'''\nimages_batches comes out in the following dim structure: (batch, quad, aug)\nI need it in (quad, aug, batch)\n'''\nls_images = [[[images_batches[i][j][k] for i in range(len(images_batches))] for k in range(len(images_batches[0][0]))] for j in range(len(images_batches[0]))]\n\nls_predictions = quad_ensemble_predictions(ls_images, NETS)\ncombined_predictions = combine_predictions(ls_predictions)    \n\ni = 1\nreconstructed_image = np.zeros(shape=(FULL_SIZE, FULL_SIZE, 3))\nreconstructed_image[:QUAD_SIZE, :QUAD_SIZE, :] = cv2.resize(ls_images[0][0][i].permute(1,2,0).cpu().numpy(), (QUAD_SIZE, QUAD_SIZE))\nreconstructed_image[:QUAD_SIZE, -QUAD_SIZE:, :] = cv2.resize(ls_images[1][0][i].permute(1,2,0).cpu().numpy(), (QUAD_SIZE, QUAD_SIZE))\nreconstructed_image[-QUAD_SIZE:, :QUAD_SIZE, :] = cv2.resize(ls_images[2][0][i].permute(1,2,0).cpu().numpy(), (QUAD_SIZE, QUAD_SIZE))\nreconstructed_image[-QUAD_SIZE:, -QUAD_SIZE:, :] = cv2.resize(ls_images[3][0][i].permute(1,2,0).cpu().numpy(), (QUAD_SIZE, QUAD_SIZE))\n\nboxes, scores, labels = run_wbf(combined_predictions, image_index=i)\nboxes = boxes.astype(np.int32).clip(min=0, max=1024)\n\nfig, ax = plt.subplots(1, 1, figsize=(16, 10))\n\nfor box in boxes:\n    cv2.rectangle(reconstructed_image, (box[0], box[1]), (box[2], box[3]), (0, 1, 0), 2)\n    \nax.set_axis_off()\nax.imshow(reconstructed_image);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\n\nfor images_batches, image_ids, do_stitch_edges in data_loader:\n    # unpack batch dimension and repack into list of batches\n    ls_images = [[[images_batches[i][j][k] for i in range(len(images_batches))] for k in range(len(images_batches[0][0]))] for j in range(len(images_batches[0]))]\n    \n    ls_predictions = quad_ensemble_predictions(ls_images, NETS)\n    combined_predictions = combine_predictions(ls_predictions)  \n    \n    for i in range(len(images_batches)):\n        boxes, scores, labels = run_wbf(combined_predictions, image_index=i)\n        boxes = boxes.astype(np.int32).clip(min=0, max=FULL_SIZE)\n        image_id = image_ids[i]\n        \n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n        result = {\n            'image_id': image_id,\n            'PredictionString': format_prediction_string(boxes, scores)\n        }\n        results.append(result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}