{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Global Wheat Detection - Pseudo-labelling\n\nYou can get the training scripts [here](https://github.com/ufownl/global-wheat-detection).\n\n* YOLOv3 from [GluonCV](https://gluon-cv.mxnet.io/)\n* Use Darknet53 backbone\n* Use [WBF](https://github.com/ZFTurbo/Weighted-Boxes-Fusion) over TTA\n* Use pseudo-labeling technique","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Dataset","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport copy\nimport json\nimport random\nimport numpy as np\nimport mxnet as mx\nimport pandas as pd\nimport gluoncv as gcv\nfrom multiprocessing import cpu_count\nfrom multiprocessing.dummy import Pool\n\n\ndef load_dataset(root):\n    csv = pd.read_csv(os.path.join(root, \"train.csv\"))\n    data = {}\n    for i in csv.index:\n        key = csv[\"image_id\"][i]\n        bbox = json.loads(csv[\"bbox\"][i])\n        bbox = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3], 0.0]\n        if key in data:\n            data[key].append(bbox)\n        else:\n            data[key] = [bbox]\n    return sorted(\n        [(k, os.path.join(root, \"train\", k + \".jpg\"), v) for k, v in data.items()],\n        key=lambda x: x[0]\n    )\n\ndef load_image(path):\n    with open(path, \"rb\") as f:\n        buf = f.read()\n    return mx.image.imdecode(buf)\n\ndef get_batches(dataset, batch_size, width=512, height=512, net=None, ctx=mx.cpu()):\n    batches = len(dataset) // batch_size\n    sampler = Sampler(dataset, width, height, net)\n    stack_fn = [gcv.data.batchify.Stack()]\n    pad_fn = [gcv.data.batchify.Pad(pad_val=-1)]\n    if net is None:\n        batchify_fn = gcv.data.batchify.Tuple(*(stack_fn + pad_fn))\n    else:\n        batchify_fn = gcv.data.batchify.Tuple(*(stack_fn * 6 + pad_fn))\n    with Pool(cpu_count() * 2) as p:\n        for i in range(batches):\n            start = i * batch_size\n            samples = p.map(sampler, range(start, start + batch_size))\n            batch = batchify_fn(samples)\n            yield [x.as_in_context(ctx) for x in batch]\n\ndef gauss_blur(image, level):\n    return cv2.blur(image, (level * 2 + 1, level * 2 + 1))\n\ndef gauss_noise(image):\n    for i in range(image.shape[2]):\n        c = image[:, :, i]\n        diff = 255 - c.max();\n        noise = np.random.normal(0, random.randint(1, 6), c.shape)\n        noise = (noise - noise.min()) / (noise.max() - noise.min())\n        noise = diff * noise\n        image[:, :, i] = c + noise.astype(np.uint8)\n    return image\n\n\nclass YOLO3TrainTransform:\n    def __init__(self, width, height, net, mean=(0.485, 0.456, 0.406),\n                 std=(0.229, 0.224, 0.225), **kwargs):\n        self._width = width\n        self._height = height\n        self._mean = mean\n        self._std = std\n\n        # in case network has reset_ctx to gpu\n        self._fake_x = mx.nd.zeros((1, 3, height, width))\n        net = copy.deepcopy(net)\n        net.collect_params().reset_ctx(None)\n        with mx.autograd.train_mode():\n            _, self._anchors, self._offsets, self._feat_maps, _, _, _, _ = net(self._fake_x)\n        self._target_generator = gcv.model_zoo.yolo.yolo_target.YOLOV3PrefetchTargetGenerator(\n            num_class=len(net.classes), **kwargs)\n\n    def __call__(self, img, label):\n        # random expansion with prob 0.5\n        if np.random.uniform(0, 1) > 0.5:\n            img, expand = gcv.data.transforms.image.random_expand(img, max_ratio=1.5, fill=114, keep_ratio=False)\n            bbox = gcv.data.transforms.bbox.translate(label, x_offset=expand[0], y_offset=expand[1])\n        else:\n            img, bbox = img, label\n\n        # random cropping\n        h, w, _ = img.shape\n        bbox, crop = gcv.data.transforms.experimental.bbox.random_crop_with_constraints(bbox, (w, h))\n        x0, y0, w, h = crop\n        img = mx.image.fixed_crop(img, x0, y0, w, h)\n\n        # resize with random interpolation\n        h, w, _ = img.shape\n        interp = np.random.randint(0, 5)\n        img = gcv.data.transforms.image.imresize(img, self._width, self._height, interp=interp)\n        bbox = gcv.data.transforms.bbox.resize(bbox, (w, h), (self._width, self._height))\n\n        # random horizontal&vertical flip\n        h, w, _ = img.shape\n        img, flips = gcv.data.transforms.image.random_flip(img, px=0.5, py=0.5)\n        bbox = gcv.data.transforms.bbox.flip(bbox, (w, h), flip_x=flips[0], flip_y=flips[1])\n\n        # random color jittering\n        img = gcv.data.transforms.experimental.image.random_color_distort(img)\n\n        # to tensor\n        img = mx.nd.image.to_tensor(img)\n        img = mx.nd.image.normalize(img, mean=self._mean, std=self._std)\n\n        # generate training target so cpu workers can help reduce the workload on gpu\n        gt_bboxes = mx.nd.array(bbox[np.newaxis, :, :4])\n        gt_ids = mx.nd.array(bbox[np.newaxis, :, 4:5])\n        gt_mixratio = mx.nd.array(bbox[np.newaxis, :, -1:])\n        objectness, center_targets, scale_targets, weights, class_targets = self._target_generator(\n            self._fake_x, self._feat_maps, self._anchors, self._offsets,\n            gt_bboxes, gt_ids, gt_mixratio)\n        return (img, objectness[0], center_targets[0], scale_targets[0], weights[0],\n                class_targets[0], gt_bboxes[0])\n\n\nclass Sampler:\n    def __init__(self, dataset, width, height, net=None, **kwargs):\n        self._dataset = dataset\n        if net is None:\n            self._training_mode = False\n            self._transform = gcv.data.transforms.presets.yolo.YOLO3DefaultValTransform(width, height, **kwargs)\n        else:\n            self._training_mode = True\n            self._transform = YOLO3TrainTransform(width, height, net, **kwargs)\n\n    def __call__(self, idx):\n        if self._training_mode:\n            raw, bboxes = self._load_mixup(idx)\n            raw = raw.asnumpy()\n            blur = random.randint(0, 3)\n            if blur > 0:\n                raw = gauss_blur(raw, blur)\n            raw = gauss_noise(raw)\n            h, w, _ = raw.shape\n            rot = random.randint(0, 3)\n            if rot > 0:\n                raw = np.rot90(raw, k=rot)\n                if rot == 1:\n                    raw_bboxes = bboxes.copy()\n                    bboxes[:, [0, 2]] = raw_bboxes[:, [1, 3]]\n                    bboxes[:, [1, 3]] = w - raw_bboxes[:, [2, 0]]\n                elif rot == 2:\n                    bboxes[:, [0, 1, 2, 3]] = np.array([[w, h, w, h]]) - bboxes[:, [2, 3, 0, 1]]\n                elif rot == 3:\n                    raw_bboxes = bboxes.copy()\n                    bboxes[:, [0, 2]] = h - raw_bboxes[:, [1, 3]]\n                    bboxes[:, [1, 3]] = raw_bboxes[:, [2, 0]]\n                raw_bboxes = bboxes.copy()\n                bboxes[:, 0] = np.min(raw_bboxes[:, [0, 2]], axis=1)\n                bboxes[:, 1] = np.min(raw_bboxes[:, [1, 3]], axis=1)\n                bboxes[:, 2] = np.max(raw_bboxes[:, [0, 2]], axis=1)\n                bboxes[:, 3] = np.max(raw_bboxes[:, [1, 3]], axis=1)\n            raw = mx.nd.array(raw)\n        else:\n            raw = load_image(self._dataset[idx][1])\n            bboxes = np.array(self._dataset[idx][2])\n        res = self._transform(raw, bboxes)\n        return [mx.nd.array(x) for x in res]\n\n    def _load_mixup(self, idx1):\n        r = random.gauss(0.5, 0.5 / 1.96)\n        if r > 0.0:\n            raw1 = load_image(self._dataset[idx1][1])\n            bboxes1 = np.array(self._dataset[idx1][2])\n            if r >= 1.0:\n                return raw1, np.hstack([bboxes1, np.full((bboxes1.shape[0], 1), 1.0)])\n        idx2 = random.randint(0, len(self._dataset) - 1)\n        raw2 = load_image(self._dataset[idx2][1])\n        bboxes2 = np.array(self._dataset[idx2][2])\n        if r <= 0.0:\n            return raw2, np.hstack([bboxes2, np.full((bboxes2.shape[0], 1), 1.0)])\n        h = max(raw1.shape[0], raw2.shape[0])\n        w = max(raw1.shape[1], raw2.shape[1])\n        mix_raw = mx.nd.zeros(shape=(h, w, 3), dtype=\"float32\")\n        mix_raw[:raw1.shape[0], :raw1.shape[1], :] += raw1.astype(\"float32\") * r\n        mix_raw[:raw2.shape[0], :raw2.shape[1], :] += raw2.astype(\"float32\") * (1.0 - r)\n        mix_bboxes = np.vstack([\n            np.hstack([bboxes1, np.full((bboxes1.shape[0], 1), r)]),\n            np.hstack([bboxes2, np.full((bboxes2.shape[0], 1), 1.0 - r)])\n        ])\n        return mix_raw.astype(\"uint8\"), mix_bboxes\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import mxnet as mx\nimport gluoncv as gcv\n\n\ndef load_model(path, ctx=mx.cpu()):\n    net = gcv.model_zoo.yolo3_darknet53_custom([\"wheat\"], pretrained_base=False)\n    net.set_nms(post_nms=150)\n    net.load_parameters(path, ctx=ctx)\n    return net\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!ln -snf /kaggle/input/weighted-boxes-fusion/ensemble_boxes && ls -lh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport numpy as np\nimport gluoncv as gcv\nfrom ensemble_boxes import *\n\n\ndef inference(path):\n    raw = load_image(path)\n    rh, rw, _ = raw.shape\n    classes_list = []\n    scores_list = []\n    bboxes_list = []\n    for _ in range(5):\n        img, flips = gcv.data.transforms.image.random_flip(raw, px=0.5, py=0.5)\n        x, _ = gcv.data.transforms.presets.yolo.transform_test(img, short=img_s)\n        _, _, xh, xw = x.shape\n        rot = random.randint(0, 3)\n        if rot > 0:\n            x = np.rot90(x.asnumpy(), k=rot, axes=(2, 3))\n        classes, scores, bboxes = model(mx.nd.array(x, ctx=context))\n        if rot > 0:\n            if rot == 1:\n                raw_bboxes = bboxes.copy()\n                bboxes[0, :, [0, 2]] = xh - raw_bboxes[0, :, [1, 3]]\n                bboxes[0, :, [1, 3]] = raw_bboxes[0, :, [2, 0]]\n            elif rot == 2:\n                bboxes[0, :, [0, 1, 2, 3]] = mx.nd.array([[xw], [xh], [xw], [xh]], ctx=context) - bboxes[0, :, [2, 3, 0, 1]]\n            elif rot == 3:\n                raw_bboxes = bboxes.copy()\n                bboxes[0, :, [0, 2]] = raw_bboxes[0, :, [1, 3]]\n                bboxes[0, :, [1, 3]] = xw - raw_bboxes[0, :, [2, 0]]\n            raw_bboxes = bboxes.copy()\n            bboxes[0, :, 0] = raw_bboxes[0, :, [0, 2]].min(axis=0)\n            bboxes[0, :, 1] = raw_bboxes[0, :, [1, 3]].min(axis=0)\n            bboxes[0, :, 2] = raw_bboxes[0, :, [0, 2]].max(axis=0)\n            bboxes[0, :, 3] = raw_bboxes[0, :, [1, 3]].max(axis=0)\n        bboxes[0, :, :] = gcv.data.transforms.bbox.flip(bboxes[0, :, :], (xw, xh), flip_x=flips[0], flip_y=flips[1])\n        bboxes[0, :, 0::2] = (bboxes[0, :, 0::2] / (xw - 1)).clip(0.0, 1.0)\n        bboxes[0, :, 1::2] = (bboxes[0, :, 1::2] / (xh - 1)).clip(0.0, 1.0)\n        classes_list.append([\n            int(classes[0, i].asscalar()) for i in range(classes.shape[1])\n                if classes[0, i].asscalar() >= 0.0\n\n        ])\n        scores_list.append([\n            scores[0, i].asscalar() for i in range(classes.shape[1])\n                if classes[0, i].asscalar() >= 0.0\n\n        ])\n        bboxes_list.append([\n            bboxes[0, i].asnumpy().tolist() for i in range(classes.shape[1])\n                if classes[0, i].asscalar() >= 0.0\n        ])\n    bboxes, scores, classes = weighted_boxes_fusion(bboxes_list, scores_list, classes_list)\n    bboxes[:, 0::2] *= rw - 1\n    bboxes[:, 1::2] *= rh - 1\n    return bboxes, scores, classes\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport time\nimport random\nimport mxnet as mx\nimport pandas as pd\n\nmax_epochs = 15\nlearning_rate = 0.001\nbatch_size = 8\nimg_s = 512\nthreshold = 0.1\ncontext = mx.gpu()\n\nprint(\"Loading model...\")\n# model = load_model(\"/kaggle/input/global-wheat-detection-yolo-darknet-weights/global-wheat-yolo3-darknet53.params\", ctx=context)\nmodel = load_model(\"/kaggle/input/yolo3-darknet-weights/global-wheat-yolo3-darknet53.params\", ctx=context)\n\nprint(\"Loading test images...\")\ntest_images = [\n    (os.path.join(dirname, filename), os.path.splitext(filename)[0])\n        for dirname, _, filenames in os.walk('/kaggle/input/global-wheat-detection/test') for filename in filenames\n]\n\nprint(\"Pseudo labelling...\")\npseudo_set = []\nfor path, image_id in test_images:\n    print(path)\n    bboxes, scores, classes = inference(path)\n    label = [\n        [round(x) for x in bboxes[i].tolist()] + [0.0] for i in range(classes.shape[0])\n            if model.classes[int(classes[i])] == \"wheat\" and scores[i] > threshold\n    ]\n    if len(label) > 0:\n        pseudo_set.append((image_id, path, label))\n    \nprint(\"Loading training set...\")\ndataset = load_dataset(\"/kaggle/input/global-wheat-detection\")\nsplit = int(len(dataset) * 0.9)\ntraining_set = dataset[:split] + pseudo_set\nprint(\"Training set: \", len(training_set))\nvalidation_set = dataset[split:]\nprint(\"Validation set: \", len(validation_set))\n\nprint(\"Re-training...\")\ntrainer = mx.gluon.Trainer(model.collect_params(), \"Nadam\", {\n    \"learning_rate\": learning_rate\n})\nmetrics = [gcv.utils.metrics.VOCMApMetric(iou_thresh=iou) for iou in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]]\nbest_score = 0.0\n# for epoch in range(max_epochs):\n#     ts = time.time()\n#     random.shuffle(training_set)\n#     training_total_L = 0.0\n#     training_batches = 0\n#     for x, objectness, center_targets, scale_targets, weights, class_targets, gt_bboxes in get_batches(training_set, batch_size, width=img_s, height=img_s, net=model, ctx=context):\n#         training_batches += 1\n#         with mx.autograd.record():\n#             obj_loss, center_loss, scale_loss, cls_loss = model(x, gt_bboxes, objectness, center_targets, scale_targets, weights, class_targets)\n#             L = obj_loss + center_loss + scale_loss + cls_loss\n#             L.backward()\n#         trainer.step(x.shape[0])\n#         training_batch_L = mx.nd.mean(L).asscalar()\n#         if training_batch_L != training_batch_L:\n#             raise ValueError()\n#         training_total_L += training_batch_L\n#         print(\"[Epoch %d  Batch %d]  batch_loss %.10f  average_loss %.10f  elapsed %.2fs\" % (\n#             epoch, training_batches, training_batch_L, training_total_L / training_batches, time.time() - ts\n#         ))\n#     training_avg_L = training_total_L / training_batches\n#     for metric in metrics:\n#         metric.reset()\n#     for x, label in get_batches(validation_set, batch_size, width=img_s, height=img_s, ctx=context):\n#         classes, scores, bboxes = model(x)\n#         for metric in metrics:\n#             metric.update(\n#                 bboxes,\n#                 classes.reshape((0, -1)),\n#                 scores.reshape((0, -1)),\n#                 label[:, :, :4],\n#                 label[:, :, 4:5].reshape((0, -1))\n#             )\n#     score = mx.nd.array([metric.get()[1] for metric in metrics], ctx=context).mean()\n#     print(\"[Epoch %d]  training_loss %.10f  validation_score %.10f  best_score %.10f  duration %.2fs\" % (\n#         epoch + 1, training_avg_L, score.asscalar(), best_score, time.time() - ts\n#     ))\n#     if score.asscalar() > best_score:\n#         best_score = score.asscalar()\n#         model.save_parameters(\"global-wheat-yolo3-darknet53.params\")\n        \n# print(\"Loading re-trained model...\")\n# model = load_model(\"global-wheat-yolo3-darknet53.params\", ctx=context)\n\nprint(\"Inference...\")\nresults = []\nfor path, image_id in test_images:\n    print(path)\n    bboxes, scores, classes = inference(path)\n    bboxes[:, 2:4] -= bboxes[:, 0:2]\n    results.append({\n        \"image_id\": image_id,\n        \"PredictionString\": \" \".join([\n            \" \".join([str(x) for x in [scores[i]] + [round(x) for x in bboxes[i].tolist()]])\n                for i in range(classes.shape[0])\n                    if model.classes[int(classes[i])] == \"wheat\" and scores[i] > threshold\n        ])\n    })\npd.DataFrame(results, columns=['image_id', 'PredictionString']).to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}