{"cells":[{"metadata":{},"cell_type":"markdown","source":"# YOLOv4 Inference notebook\nIn this notebook, I am going to port yolov4 into kaggle notebook (it can also run yolov3 and lower version) from this [awesome git repo](https://github.com/ultralytics/yolov3)\n\nI hope it will save somebody's time.\nThis is just a quick testing, you can play around with parameters to get a better scores. \n\nI am working on porting my training script. Hope I will finish it soon.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport math\nimport os\nimport random\nimport shutil\nimport subprocess\nfrom pathlib import Path\n\nimport cv2\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-deps '../input/weightedboxesfusion/' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Layers\nDefine some layers for yolov4","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn.functional as F\n\n\ndef make_divisible(v, divisor):\n    # Function ensures all layers have a channel number that is divisible by 8\n    # https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py\n    return math.ceil(v / divisor) * divisor\n\n\nclass Flatten(nn.Module):\n    # Use after nn.AdaptiveAvgPool2d(1) to remove last 2 dimensions\n    def forward(self, x):\n        return x.view(x.size(0), -1)\n\n\nclass Concat(nn.Module):\n    # Concatenate a list of tensors along dimension\n    def __init__(self, dimension=1):\n        super(Concat, self).__init__()\n        self.d = dimension\n\n    def forward(self, x):\n        return torch.cat(x, self.d)\n\n\nclass FeatureConcat(nn.Module):\n    def __init__(self, layers):\n        super(FeatureConcat, self).__init__()\n        self.layers = layers  # layer indices\n        self.multiple = len(layers) > 1  # multiple layers flag\n\n    def forward(self, x, outputs):\n        return torch.cat([outputs[i] for i in self.layers], 1) if self.multiple else outputs[self.layers[0]]\n\n\nclass WeightedFeatureFusion(nn.Module):  # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n    def __init__(self, layers, weight=False):\n        super(WeightedFeatureFusion, self).__init__()\n        self.layers = layers  # layer indices\n        self.weight = weight  # apply weights boolean\n        self.n = len(layers) + 1  # number of layers\n        if weight:\n            self.w = nn.Parameter(torch.zeros(self.n), requires_grad=True)  # layer weights\n\n    def forward(self, x, outputs):\n        # Weights\n        if self.weight:\n            w = torch.sigmoid(self.w) * (2 / self.n)  # sigmoid weights (0-1)\n            x = x * w[0]\n\n        # Fusion\n        nx = x.shape[1]  # input channels\n        for i in range(self.n - 1):\n            a = outputs[self.layers[i]] * w[i + 1] if self.weight else outputs[self.layers[i]]  # feature to add\n            na = a.shape[1]  # feature channels\n\n            # Adjust channels\n            if nx == na:  # same shape\n                x = x + a\n            elif nx > na:  # slice input\n                x[:, :na] = x[:, :na] + a  # or a = nn.ZeroPad2d((0, 0, 0, 0, 0, dc))(a); x = x + a\n            else:  # slice feature\n                x = x + a[:, :nx]\n\n        return x\n\n\nclass MixConv2d(nn.Module):  # MixConv: Mixed Depthwise Convolutional Kernels https://arxiv.org/abs/1907.09595\n    def __init__(self, in_ch, out_ch, k=(3, 5, 7), stride=1, dilation=1, bias=True, method='equal_params'):\n        super(MixConv2d, self).__init__()\n\n        groups = len(k)\n        if method == 'equal_ch':  # equal channels per group\n            i = torch.linspace(0, groups - 1E-6, out_ch).floor()  # out_ch indices\n            ch = [(i == g).sum() for g in range(groups)]\n        else:  # 'equal_params': equal parameter count per group\n            b = [out_ch] + [0] * groups\n            a = np.eye(groups + 1, groups, k=-1)\n            a -= np.roll(a, 1, axis=1)\n            a *= np.array(k) ** 2\n            a[0] = 1\n            ch = np.linalg.lstsq(a, b, rcond=None)[0].round().astype(int)  # solve for equal weight indices, ax = b\n\n        self.m = nn.ModuleList([nn.Conv2d(in_channels=in_ch,\n                                          out_channels=ch[g],\n                                          kernel_size=k[g],\n                                          stride=stride,\n                                          padding=k[g] // 2,  # 'same' pad\n                                          dilation=dilation,\n                                          bias=bias) for g in range(groups)])\n\n    def forward(self, x):\n        return torch.cat([m(x) for m in self.m], 1)\n\n\n# Activation functions below -------------------------------------------------------------------------------------------\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x * torch.sigmoid(x)\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_tensors[0]\n        sx = torch.sigmoid(x)  # sigmoid(ctx)\n        return grad_output * (sx * (1 + x * (1 - sx)))\n\n\nclass MishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x.mul(torch.tanh(F.softplus(x)))  # x * tanh(ln(1 + exp(x)))\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_tensors[0]\n        sx = torch.sigmoid(x)\n        fx = F.softplus(x).tanh()\n        return grad_output * (fx + x * sx * (1 - fx * fx))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\n\nclass MemoryEfficientMish(nn.Module):\n    def forward(self, x):\n        return MishImplementation.apply(x)\n\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\nclass HardSwish(nn.Module):  # https://arxiv.org/pdf/1905.02244.pdf\n    def forward(self, x):\n        return x * F.hardtanh(x + 3, 0., 6., True) / 6.\n\n\nclass Mish(nn.Module):  # https://github.com/digantamisra98/Mish\n    def forward(self, x):\n        return x * F.softplus(x).tanh()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parsing yolov4 config file\nRead config file and create model.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\ndef parse_data_cfg(path):\n    # Parses the data configuration file\n    if not os.path.exists(path) and os.path.exists('data' + os.sep + path):  # add data/ prefix if omitted\n        path = 'data' + os.sep + path\n\n    with open(path, 'r') as f:\n        lines = f.readlines()\n\n    options = dict()\n    for line in lines:\n        line = line.strip()\n        if line == '' or line.startswith('#'):\n            continue\n        key, val = line.split('=')\n        options[key.strip()] = val.strip()\n\n    return options\n\ndef parse_model_cfg(path):\n    # Parse the yolo *.cfg file and return module definitions path may be 'cfg/yolov3.cfg', 'yolov3.cfg', or 'yolov3'\n    if not path.endswith('.cfg'):  # add .cfg suffix if omitted\n        path += '.cfg'\n    if not os.path.exists(path) and os.path.exists('cfg' + os.sep + path):  # add cfg/ prefix if omitted\n        path = 'cfg' + os.sep + path\n\n    with open(path, 'r') as f:\n        lines = f.read().split('\\n')\n    lines = [x for x in lines if x and not x.startswith('#')]\n    lines = [x.rstrip().lstrip() for x in lines]  # get rid of fringe whitespaces\n    mdefs = []  # module definitions\n    for line in lines:\n        if line.startswith('['):  # This marks the start of a new block\n            mdefs.append({})\n            mdefs[-1]['type'] = line[1:-1].rstrip()\n            if mdefs[-1]['type'] == 'convolutional':\n                mdefs[-1]['batch_normalize'] = 0  # pre-populate with zeros (may be overwritten later)\n        else:\n            key, val = line.split(\"=\")\n            key = key.rstrip()\n\n            if key == 'anchors':  # return nparray\n                mdefs[-1][key] = np.array([float(x) for x in val.split(',')]).reshape((-1, 2))  # np anchors\n            elif (key in ['from', 'layers', 'mask']) or (key == 'size' and ',' in val):  # return array\n                mdefs[-1][key] = [int(x) for x in val.split(',')]\n            else:\n                val = val.strip()\n                if val.isnumeric():  # return int or float\n                    mdefs[-1][key] = int(val) if (int(val) - float(val)) == 0 else float(val)\n                else:\n                    mdefs[-1][key] = val  # return string\n\n    # Check all fields are supported\n    supported = ['type', 'batch_normalize', 'filters', 'size', 'stride', 'pad', 'activation', 'layers', 'groups',\n                 'from', 'mask', 'anchors', 'classes', 'num', 'jitter', 'ignore_thresh', 'truth_thresh', 'random',\n                 'stride_x', 'stride_y', 'weights_type', 'weights_normalization', 'scale_x_y', 'beta_nms', 'nms_kind',\n                 'iou_loss', 'iou_normalizer', 'cls_normalizer', 'iou_thresh', 'stopbackward', 'max_delta']\n\n    f = []  # fields\n    for x in mdefs[1:]:\n        [f.append(k) for k in x if k not in f]\n    u = [x for x in f if x not in supported]  # unsupported fields\n    assert not any(u), \"Unsupported fields %s in %s. See https://github.com/ultralytics/yolov3/issues/631\" % (u, path)\n\n    return mdefs\n\ndef scale_img(img, ratio=1.0, same_shape=True):  # img(16,3,256,416), r=ratio\n    # scales img(bs,3,y,x) by ratio\n    h, w = img.shape[2:]\n    s = (int(h * ratio), int(w * ratio))  # new size\n    img = F.interpolate(img, size=s, mode='bilinear', align_corners=False)  # resize\n    if not same_shape:  # pad/crop img\n        gs = 64  # (pixels) grid size\n        h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)]\n    return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447)  # value = imagenet mean\n\ndef create_modules(module_defs, img_size, cfg):\n    # Constructs module list of layer blocks from module configuration in module_defs\n\n    img_size = [img_size] * 2 if isinstance(img_size, int) else img_size  # expand if necessary\n    _ = module_defs.pop(0)  # cfg training hyperparams (unused)\n    output_filters = [3]  # input channels\n    module_list = nn.ModuleList()\n    routs = []  # list of layers which rout to deeper layers\n    yolo_index = -1\n\n    for i, mdef in enumerate(module_defs):\n        modules = nn.Sequential()\n\n        if mdef['type'] == 'convolutional':\n            bn = mdef['batch_normalize']\n            filters = mdef['filters']\n            k = mdef['size']  # kernel size\n            stride = mdef['stride'] if 'stride' in mdef else (mdef['stride_y'], mdef['stride_x'])\n            if isinstance(k, int):  # single-size conv\n                modules.add_module('Conv2d', nn.Conv2d(in_channels=output_filters[-1],\n                                                       out_channels=filters,\n                                                       kernel_size=k,\n                                                       stride=stride,\n                                                       padding=k // 2 if mdef['pad'] else 0,\n                                                       groups=mdef['groups'] if 'groups' in mdef else 1,\n                                                       bias=not bn))\n            else:  # multiple-size conv\n                modules.add_module('MixConv2d', MixConv2d(in_ch=output_filters[-1],\n                                                          out_ch=filters,\n                                                          k=k,\n                                                          stride=stride,\n                                                          bias=not bn))\n\n            if bn:\n                modules.add_module('BatchNorm2d', nn.BatchNorm2d(filters, momentum=0.03, eps=1E-4))\n            else:\n                routs.append(i)  # detection output (goes into yolo layer)\n\n            if mdef['activation'] == 'leaky':  # activation study https://github.com/ultralytics/yolov3/issues/441\n                modules.add_module('activation', nn.LeakyReLU(0.1, inplace=True))\n            elif mdef['activation'] == 'swish':\n                modules.add_module('activation', Swish())\n            elif mdef['activation'] == 'mish':\n                modules.add_module('activation', Mish())\n\n        elif mdef['type'] == 'BatchNorm2d':\n            filters = output_filters[-1]\n            modules = nn.BatchNorm2d(filters, momentum=0.03, eps=1E-4)\n            if i == 0 and filters == 3:  # normalize RGB image\n                # imagenet mean and var https://pytorch.org/docs/stable/torchvision/models.html#classification\n                modules.running_mean = torch.tensor([0.485, 0.456, 0.406])\n                modules.running_var = torch.tensor([0.0524, 0.0502, 0.0506])\n\n        elif mdef['type'] == 'maxpool':\n            k = mdef['size']  # kernel size\n            stride = mdef['stride']\n            maxpool = nn.MaxPool2d(kernel_size=k, stride=stride, padding=(k - 1) // 2)\n            if k == 2 and stride == 1:  # yolov3-tiny\n                modules.add_module('ZeroPad2d', nn.ZeroPad2d((0, 1, 0, 1)))\n                modules.add_module('MaxPool2d', maxpool)\n            else:\n                modules = maxpool\n\n        elif mdef['type'] == 'upsample':\n            modules = nn.Upsample(scale_factor=mdef['stride'])\n\n        elif mdef['type'] == 'route':  # nn.Sequential() placeholder for 'route' layer\n            layers = mdef['layers']\n            filters = sum([output_filters[l + 1 if l > 0 else l] for l in layers])\n            routs.extend([i + l if l < 0 else l for l in layers])\n            modules = FeatureConcat(layers=layers)\n\n        elif mdef['type'] == 'shortcut':  # nn.Sequential() placeholder for 'shortcut' layer\n            layers = mdef['from']\n            filters = output_filters[-1]\n            routs.extend([i + l if l < 0 else l for l in layers])\n            modules = WeightedFeatureFusion(layers=layers, weight='weights_type' in mdef)\n\n        elif mdef['type'] == 'reorg3d':  # yolov3-spp-pan-scale\n            pass\n\n        elif mdef['type'] == 'yolo':\n            yolo_index += 1\n            stride = [32, 16, 8]  # P5, P4, P3 strides\n            if 'panet' in cfg or 'yolov4' in cfg:  # stride order reversed\n                stride = list(reversed(stride))\n            layers = mdef['from'] if 'from' in mdef else []\n            modules = YOLOLayer(anchors=mdef['anchors'][mdef['mask']],  # anchor list\n                                nc=mdef['classes'],  # number of classes\n                                img_size=img_size,  # (416, 416)\n                                yolo_index=yolo_index,  # 0, 1, 2...\n                                layers=layers,  # output layers\n                                stride=stride[yolo_index])\n\n            # Initialize preceding Conv2d() bias (https://arxiv.org/pdf/1708.02002.pdf section 3.3)\n            try:\n                j = layers[yolo_index] if 'from' in mdef else -1\n                bias_ = module_list[j][0].bias  # shape(255,)\n                bias = bias_[:modules.no * modules.na].view(modules.na, -1)  # shape(3,85)\n                bias[:, 4] += -4.5  # obj\n                bias[:, 5:] += math.log(0.6 / (modules.nc - 0.99))  # cls (sigmoid(p) = 1/nc)\n                module_list[j][0].bias = torch.nn.Parameter(bias_, requires_grad=bias_.requires_grad)\n            except:\n                print('WARNING: smart bias initialization failure.')\n\n        else:\n            print('Warning: Unrecognized Layer Type: ' + mdef['type'])\n\n        # Register module list and number of output filters\n        module_list.append(modules)\n        output_filters.append(filters)\n\n    routs_binary = [False] * (i + 1)\n    for i in routs:\n        routs_binary[i] = True\n    return module_list, routs_binary\n\n\nclass YOLOLayer(nn.Module):\n    def __init__(self, anchors, nc, img_size, yolo_index, layers, stride):\n        super(YOLOLayer, self).__init__()\n        self.anchors = torch.Tensor(anchors)\n        self.index = yolo_index  # index of this layer in layers\n        self.layers = layers  # model output layer indices\n        self.stride = stride  # layer stride\n        self.nl = len(layers)  # number of output layers (3)\n        self.na = len(anchors)  # number of anchors (3)\n        self.nc = nc  # number of classes (80)\n        self.no = nc + 5  # number of outputs (85)\n        self.nx, self.ny, self.ng = 0, 0, 0  # initialize number of x, y gridpoints\n        self.anchor_vec = self.anchors / self.stride\n        self.anchor_wh = self.anchor_vec.view(1, self.na, 1, 1, 2)\n\n\n    def create_grids(self, ng=(13, 13), device='cpu'):\n        self.nx, self.ny = ng  # x and y grid size\n        self.ng = torch.tensor(ng, dtype=torch.float)\n\n        # build xy offsets\n        if not self.training:\n            yv, xv = torch.meshgrid([torch.arange(self.ny, device=device), torch.arange(self.nx, device=device)])\n            self.grid = torch.stack((xv, yv), 2).view((1, 1, self.ny, self.nx, 2)).float()\n\n        if self.anchor_vec.device != device:\n            self.anchor_vec = self.anchor_vec.to(device)\n            self.anchor_wh = self.anchor_wh.to(device)\n\n    def forward(self, p, out):\n        ASFF = False  # https://arxiv.org/abs/1911.09516\n        if ASFF:\n            i, n = self.index, self.nl  # index in layers, number of layers\n            p = out[self.layers[i]]\n            bs, _, ny, nx = p.shape  # bs, 255, 13, 13\n            if (self.nx, self.ny) != (nx, ny):\n                self.create_grids((nx, ny), p.device)\n\n            # outputs and weights\n            # w = F.softmax(p[:, -n:], 1)  # normalized weights\n            w = torch.sigmoid(p[:, -n:]) * (2 / n)  # sigmoid weights (faster)\n            # w = w / w.sum(1).unsqueeze(1)  # normalize across layer dimension\n\n            # weighted ASFF sum\n            p = out[self.layers[i]][:, :-n] * w[:, i:i + 1]\n            for j in range(n):\n                if j != i:\n                    p += w[:, j:j + 1] * \\\n                         F.interpolate(out[self.layers[j]][:, :-n], size=[ny, nx], mode='bilinear', align_corners=False)\n\n        else:\n            bs, _, ny, nx = p.shape  # bs, 255, 13, 13\n            if (self.nx, self.ny) != (nx, ny):\n                self.create_grids((nx, ny), p.device)\n\n        # p.view(bs, 255, 13, 13) -- > (bs, 3, 13, 13, 85)  # (bs, anchors, grid, grid, classes + xywh)\n        p = p.view(bs, self.na, self.no, self.ny, self.nx).permute(0, 1, 3, 4, 2).contiguous()  # prediction\n\n        if self.training:\n            return p\n\n\n        else:  # inference\n            io = p.clone()  # inference output\n            io[..., :2] = torch.sigmoid(io[..., :2]) + self.grid  # xy\n            io[..., 2:4] = torch.exp(io[..., 2:4]) * self.anchor_wh  # wh yolo method\n            io[..., :4] *= self.stride\n            torch.sigmoid_(io[..., 4:])\n            return io.view(bs, -1, self.no), p  # view [1, 3, 13, 13, 85] as [1, 507, 85]\n\n\nclass Darknet(nn.Module):\n    # YOLOv3 object detection model\n\n    def __init__(self, cfg, img_size=(416, 416), verbose=False):\n        super(Darknet, self).__init__()\n\n        self.module_defs = parse_model_cfg(cfg)\n        self.module_list, self.routs = create_modules(self.module_defs, img_size, cfg)\n        self.yolo_layers = get_yolo_layers(self)\n        # torch_utils.initialize_weights(self)\n\n        # Darknet Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346\n        self.version = np.array([0, 2, 5], dtype=np.int32)  # (int32) version info: major, minor, revision\n        self.seen = np.array([0], dtype=np.int64)  # (int64) number of images seen during training\n        #self.info(verbose) #if not ONNX_EXPORT else None  # print model description\n\n    def forward(self, x, augment=False, verbose=False):\n\n        if not augment:\n            return self.forward_once(x)\n        else:  # Augment images (inference and test only) https://github.com/ultralytics/yolov3/issues/931\n            img_size = x.shape[-2:]  # height, width\n            s = [0.83, 0.67]  # scales\n            y = []\n            for i, xi in enumerate((x,\n                                    scale_img(x.flip(3), s[0], same_shape=False),  # flip-lr and scale\n                                    scale_img(x, s[1], same_shape=False),  # scale\n                                    )):\n                # cv2.imwrite('img%g.jpg' % i, 255 * xi[0].numpy().transpose((1, 2, 0))[:, :, ::-1])\n                y.append(self.forward_once(xi)[0])\n\n            y[1][..., :4] /= s[0]  # scale\n            y[1][..., 0] = img_size[1] - y[1][..., 0]  # flip lr\n            y[2][..., :4] /= s[1]  # scale\n\n            # for i, yi in enumerate(y):  # coco small, medium, large = < 32**2 < 96**2 <\n            #     area = yi[..., 2:4].prod(2)[:, :, None]\n            #     if i == 1:\n            #         yi *= (area < 96. ** 2).float()\n            #     elif i == 2:\n            #         yi *= (area > 32. ** 2).float()\n            #     y[i] = yi\n\n            y = torch.cat(y, 1)\n            return y, None\n\n    def forward_once(self, x, augment=False, verbose=False):\n        img_size = x.shape[-2:]  # height, width\n        yolo_out, out = [], []\n        if verbose:\n            print('0', x.shape)\n            str = ''\n\n        # Augment images (inference and test only)\n        if augment:  # https://github.com/ultralytics/yolov3/issues/931\n            nb = x.shape[0]  # batch size\n            s = [0.83, 0.67]  # scales\n            x = torch.cat((x,\n                           torch_utils.scale_img(x.flip(3), s[0]),  # flip-lr and scale\n                           torch_utils.scale_img(x, s[1]),  # scale\n                           ), 0)\n\n        for i, module in enumerate(self.module_list):\n            name = module.__class__.__name__\n            if name in ['WeightedFeatureFusion', 'FeatureConcat']:  # sum, concat\n                if verbose:\n                    l = [i - 1] + module.layers  # layers\n                    sh = [list(x.shape)] + [list(out[i].shape) for i in module.layers]  # shapes\n                    str = ' >> ' + ' + '.join(['layer %g %s' % x for x in zip(l, sh)])\n                x = module(x, out)  # WeightedFeatureFusion(), FeatureConcat()\n            elif name == 'YOLOLayer':\n                yolo_out.append(module(x, out))\n            else:  # run module directly, i.e. mtype = 'convolutional', 'upsample', 'maxpool', 'batchnorm2d' etc.\n                x = module(x)\n\n            out.append(x if self.routs[i] else [])\n            if verbose:\n                print('%g/%g %s -' % (i, len(self.module_list), name), list(x.shape), str)\n                str = ''\n\n        if self.training:  # train\n            return yolo_out\n        else:  # inference or test\n            x, p = zip(*yolo_out)  # inference output, training output\n            x = torch.cat(x, 1)  # cat yolo outputs\n            if augment:  # de-augment results\n                x = torch.split(x, nb, dim=0)\n                x[1][..., :4] /= s[0]  # scale\n                x[1][..., 0] = img_size[1] - x[1][..., 0]  # flip lr\n                x[2][..., :4] /= s[1]  # scale\n                x = torch.cat(x, 1)\n            return x, p\n\n\n\ndef get_yolo_layers(model):\n    return [i for i, m in enumerate(model.module_list) if m.__class__.__name__ == 'YOLOLayer']  # [89, 101, 113]\n\n\ndef load_darknet_weights(self, weights, cutoff=-1):\n    # Parses and loads the weights stored in 'weights'\n\n    # Establish cutoffs (load layers between 0 and cutoff. if cutoff = -1 all are loaded)\n    file = Path(weights).name\n    if file == 'darknet53.conv.74':\n        cutoff = 75\n    elif file == 'yolov3-tiny.conv.15':\n        cutoff = 15\n\n    # Read weights file\n    with open(weights, 'rb') as f:\n        # Read Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346\n        self.version = np.fromfile(f, dtype=np.int32, count=3)  # (int32) version info: major, minor, revision\n        self.seen = np.fromfile(f, dtype=np.int64, count=1)  # (int64) number of images seen during training\n\n        weights = np.fromfile(f, dtype=np.float32)  # the rest are weights\n\n    ptr = 0\n    for i, (mdef, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):\n        if mdef['type'] == 'convolutional':\n            conv = module[0]\n            if mdef['batch_normalize']:\n                # Load BN bias, weights, running mean and running variance\n                bn = module[1]\n                nb = bn.bias.numel()  # number of biases\n                # Bias\n                bn.bias.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.bias))\n                ptr += nb\n                # Weight\n                bn.weight.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.weight))\n                ptr += nb\n                # Running Mean\n                bn.running_mean.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.running_mean))\n                ptr += nb\n                # Running Var\n                bn.running_var.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.running_var))\n                ptr += nb\n            else:\n                # Load conv. bias\n                nb = conv.bias.numel()\n                conv_b = torch.from_numpy(weights[ptr:ptr + nb]).view_as(conv.bias)\n                conv.bias.data.copy_(conv_b)\n                ptr += nb\n            # Load conv. weights\n            nw = conv.weight.numel()  # number of weights\n            conv.weight.data.copy_(torch.from_numpy(weights[ptr:ptr + nw]).view_as(conv.weight))\n            ptr += nw","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def detect1Image(im0, imgsz, model, device, conf_thres, iou_thres):\n    img = letterbox(im0, new_shape=imgsz)[0]\n    # Convert\n    img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416\n    img = np.ascontiguousarray(img)\n\n\n    img = torch.from_numpy(img).to(device)\n    img =  img.float()  # uint8 to fp16/32\n    img /= 255.0   \n    if img.ndimension() == 3:\n        img = img.unsqueeze(0)\n\n    # Inference\n    pred = model(img, augment=False)[0]\n\n    # Apply NMS\n    pred = non_max_suppression(pred, conf_thres, iou_thres)\n\n    # p, s, im0 = path, '', im0s\n    # print(path)\n    # Process detections\n    boxes = []\n    scores = []\n    for i, det in enumerate(pred):  # detections per image\n        # save_path = 'draw/' + image_id + '.jpg'\n        if det is not None and len(det):\n            # Rescale boxes from img_size to im0 size\n            det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round()\n\n            # Write results\n            for *xyxy, conf, cls in det:\n                #c1, c2 = (int(xyxy[0]), int(xyxy[1])), (int(xyxy[2]), int(xyxy[3]))\n                #cv2.rectangle(im0, c1, c2, (0,0,255), thickness=2)\n                #cv2.putText(im0, '%.2f'%(conf), c1, cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,255,255), 2)\n                boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])\n                scores.append(conf)\n\n    return np.array(boxes), np.array(scores) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_formats = ['.bmp', '.jpg', '.jpeg', '.png', '.tif', '.dng']\ndef letterbox(img, new_shape=(416, 416), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True):\n    # Resize image to a 32-pixel-multiple rectangle https://github.com/ultralytics/yolov3/issues/232\n    shape = img.shape[:2]  # current shape [height, width]\n    if isinstance(new_shape, int):\n        new_shape = (new_shape, new_shape)\n\n    # Scale ratio (new / old)\n    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n    if not scaleup:  # only scale down, do not scale up (for better test mAP)\n        r = min(r, 1.0)\n\n    # Compute padding\n    ratio = r, r  # width, height ratios\n    new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding\n    if auto:  # minimum rectangle\n        dw, dh = np.mod(dw, 64), np.mod(dh, 64)  # wh padding\n    elif scaleFill:  # stretch\n        dw, dh = 0.0, 0.0\n        new_unpad = new_shape\n        ratio = new_shape[0] / shape[1], new_shape[1] / shape[0]  # width, height ratios\n\n    dw /= 2  # divide padding into 2 sides\n    dh /= 2\n\n    if shape[::-1] != new_unpad:  # resize\n        img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)\n    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n    img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border\n    return img, ratio, (dw, dh)\n\nclass LoadImages:  # for inference\n    def __init__(self, path, img_size=416):\n        path = str(Path(path))  # os-agnostic\n        files = []\n        if os.path.isdir(path):\n            files = sorted(glob.glob(os.path.join(path, '*.*')))\n\n        images = [x for x in files if os.path.splitext(x)[-1].lower() in img_formats]\n        nI, nV = len(images), 0\n\n        self.img_size = img_size\n        self.files = images #+ videos\n        self.nF = nI + nV  # number of files\n        #self.video_flag = [False] * nI + [True] * nV\n        self.mode = 'images'\n        #if any(videos):\n        #    self.new_video(videos[0])  # new video\n        #else:\n        #    self.cap = None\n        assert self.nF > 0, 'No images or videos found in ' + path\n\n    def __iter__(self):\n        self.count = 0\n        return self\n\n    def __next__(self):\n        if self.count == self.nF:\n            raise StopIteration\n        path = self.files[self.count]\n\n        # Read image\n        self.count += 1\n        img0 = cv2.imread(path)  # BGR\n        assert img0 is not None, 'Image Not Found ' + path\n        print('image %g/%g %s: ' % (self.count, self.nF, path), end='')\n\n        # Padded resize\n        img = letterbox(img0, new_shape=self.img_size)[0]\n\n        # Convert\n        img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416\n        img = np.ascontiguousarray(img)\n\n        # cv2.imwrite(path + '.letterbox.jpg', 255 * img.transpose((1, 2, 0))[:, :, ::-1])  # save letterbox image\n        return path, img, img0 #, self.cap\n\n    def __len__(self):\n        return self.nF  # number of files","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Post processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def xywh2xyxy(x):\n    # Transform box coordinates from [x, y, w, h] to [x1, y1, x2, y2] (where xy1=top-left, xy2=bottom-right)\n    y = torch.zeros_like(x) if isinstance(x, torch.Tensor) else np.zeros_like(x)\n    y[:, 0] = x[:, 0] - x[:, 2] / 2  # top left x\n    y[:, 1] = x[:, 1] - x[:, 3] / 2  # top left y\n    y[:, 2] = x[:, 0] + x[:, 2] / 2  # bottom right x\n    y[:, 3] = x[:, 1] + x[:, 3] / 2  # bottom right y\n    return y\ndef non_max_suppression(prediction, conf_thres=0.1, iou_thres=0.6, multi_label=True, classes=None, agnostic=False):\n    \"\"\"\n    Performs  Non-Maximum Suppression on inference results\n    Returns detections with shape:\n        nx6 (x1, y1, x2, y2, conf, cls)\n    \"\"\"\n\n    # Box constraints\n    min_wh, max_wh = 2, 4096  # (pixels) minimum and maximum box width and height\n\n    method = 'merge'\n    nc = prediction[0].shape[1] - 5  # number of classes\n    multi_label &= nc > 1  # multiple labels per box\n    output = [None] * len(prediction)\n\n    for xi, x in enumerate(prediction):  # image index, image inference\n        # Apply conf constraint\n        x = x[x[:, 4] > conf_thres]\n\n        # Apply width-height constraint\n        x = x[((x[:, 2:4] > min_wh) & (x[:, 2:4] < max_wh)).all(1)]\n\n        # If none remain process next image\n        if not x.shape[0]:\n            continue\n\n        # Compute conf\n        x[..., 5:] *= x[..., 4:5]  # conf = obj_conf * cls_conf\n\n        # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n        box = xywh2xyxy(x[:, :4])\n\n        # Detections matrix nx6 (xyxy, conf, cls)\n        if multi_label:\n            i, j = (x[:, 5:] > conf_thres).nonzero().t()\n            x = torch.cat((box[i], x[i, j + 5].unsqueeze(1), j.float().unsqueeze(1)), 1)\n        else:  # best class only\n            conf, j = x[:, 5:].max(1)\n            x = torch.cat((box, conf.unsqueeze(1), j.float().unsqueeze(1)), 1)\n\n        # Filter by class\n        if classes:\n            x = x[(j.view(-1, 1) == torch.tensor(classes, device=j.device)).any(1)]\n\n        # Apply finite constraint\n        if not torch.isfinite(x).all():\n            x = x[torch.isfinite(x).all(1)]\n\n        # If none remain process next image\n        n = x.shape[0]  # number of boxes\n        if not n:\n            continue\n\n        # Sort by confidence\n        # if method == 'fast_batch':\n        #    x = x[x[:, 4].argsort(descending=True)]\n\n        # Batched NMS\n        c = x[:, 5] * 0 if agnostic else x[:, 5]  # classes\n        boxes, scores = x[:, :4].clone() + c.view(-1, 1) * max_wh, x[:, 4]  # boxes (offset by class), scores\n        if method == 'merge':  # Merge NMS (boxes merged using weighted mean)\n            i = torchvision.ops.boxes.nms(boxes, scores, iou_thres)\n            if 1 < n < 3E3:  # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n                try:\n                    # weights = (box_iou(boxes, boxes).tril_() > iou_thres) * scores.view(-1, 1)  # box weights\n                    # weights /= weights.sum(0)  # normalize\n                    # x[:, :4] = torch.mm(weights.T, x[:, :4])\n                    weights = (box_iou(boxes[i], boxes) > iou_thres) * scores[None]  # box weights\n                    x[i, :4] = torch.mm(weights / weights.sum(1, keepdim=True), x[:, :4]).float()  # merged boxes\n                except:  # possible CUDA error https://github.com/ultralytics/yolov3/issues/1139\n                    pass\n        elif method == 'vision':\n            i = torchvision.ops.boxes.nms(boxes, scores, iou_thres)\n        elif method == 'fast':  # FastNMS from https://github.com/dbolya/yolact\n            iou = box_iou(boxes, boxes).triu_(diagonal=1)  # upper triangular iou matrix\n            i = iou.max(0)[0] < iou_thres\n\n        output[xi] = x[i]\n    return output\n\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\n    # Rescale coords (xyxy) from img1_shape to img0_shape\n    if ratio_pad is None:  # calculate from img0_shape\n        gain = max(img1_shape) / max(img0_shape)  # gain  = old / new\n        pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2  # wh padding\n    else:\n        gain = ratio_pad[0][0]\n        pad = ratio_pad[1]\n\n    coords[:, [0, 2]] -= pad[0]  # x padding\n    coords[:, [1, 3]] -= pad[1]  # y padding\n    coords[:, :4] /= gain\n    clip_coords(coords, img0_shape)\n    return coords\n\n\ndef clip_coords(boxes, img_shape):\n    # Clip bounding xyxy bounding boxes to image shape (height, width)\n    boxes[:, 0].clamp_(0, img_shape[1])  # x1\n    boxes[:, 1].clamp_(0, img_shape[0])  # y1\n    boxes[:, 2].clamp_(0, img_shape[1])  # x2\n    boxes[:, 3].clamp_(0, img_shape[0])  # y2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert to submission format\nThis piece of code is from [this notebook](https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-inference) Thanks @peter. I have leant and used a lot of tricks from you.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Yolov4 detection\nThe most important part here. Make detection.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from ensemble_boxes import *\ndef run_wbf(boxes, scores, image_size=1023, iou_thr=0.5, skip_box_thr=0.7, weights=None):\n    #boxes = [prediction[image_index]['boxes'].data.cpu().numpy()/(image_size-1) for prediction in predictions]\n    #scores = [prediction[image_index]['scores'].data.cpu().numpy() for prediction in predictions]\n    labels = [np.zeros(score.shape[0]) for score in scores]\n    boxes = [box/(image_size) for box in boxes]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    #boxes, scores, labels = nms(boxes, scores, labels, weights=[1,1,1,1,1], iou_thr=0.5)\n    boxes = boxes*(image_size)\n    return boxes, scores, labels\n\ndef TTAImage(image, index):\n    image1 = image.copy()\n    if index==0: \n        rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image\n    elif index==1:\n        rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image2\n    elif index==2:\n        rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)\n        return rotated_image3\n    elif index == 3:\n        return image1\n    \ndef rotBoxes90(boxes, im_w, im_h):\n    ret_boxes =[]\n    for box in boxes:\n        x1, y1, x2, y2 = box\n        x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2\n        x1, y1, x2, y2 = y1, -x1, y2, -x2\n        x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)\n        x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)\n        ret_boxes.append([x1a, y1a, x2a, y2a])\n    return np.array(ret_boxes)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def detect(save_img=False):\n    img_size = 608\n    #weights = '/kaggle/input/yoloweights/yolov4.weights'\n    #cfg = '/kaggle/input/yoloweights/yolov4.cfg'\n    weights = '/kaggle/input/yolov4weights/yolov4-78k_final.weights'\n    cfg = '/kaggle/input/yolov4weights/yolov4-custom.cfg'\n    testdir = '/kaggle/input/global-wheat-detection/test'\n    conf_thres = 0.3\n    iou_thres= 0.6\n    # Initialize\n    #device = torch_utils.select_device(device='cpu' if ONNX_EXPORT else opt.device)\n    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\n    # Initialize model\n    model = Darknet(cfg, img_size)\n\n    # Load weights\n    if weights.endswith('.pt'):  # pytorch format\n        model.load_state_dict(torch.load(weights, map_location=device)['model'])\n    else:  # darknet format\n        load_darknet_weights(model, weights)\n\n\n    # Eval mode\n    model.to(device).eval()\n\n    imagenames =  os.listdir(testdir)\n    # Set Dataloader\n    save_img = True\n    #dataset = LoadImages(testdir, img_size=img_size)\n\n    # Run inference\n    #img = torch.zeros((1, 3, img_size, img_size), device=device)  # init img\n    #_ = model(img.float()) if device.type != 'cpu' else None  # run once\n    \n    results = []\n    fig, ax = plt.subplots(5, 2, figsize=(30, 80))\n    count = 0\n    for name in imagenames:\n        image_id = name.split('.')[0]\n        im01 = cv2.imread('%s/%s.jpg'%(testdir,image_id))  # BGR\n        assert im01 is not None, 'Image Not Found '\n        # Padded resize\n        im_w, im_h = im01.shape[:2]\n        enboxes = []\n        enscores = []\n        if 1:\n            for i in range(4):\n                im0 = TTAImage(im01, i)\n                boxes, scores = detect1Image(im0, img_size, model, device, conf_thres, iou_thres)\n                for _ in range(3-i):\n                    boxes = rotBoxes90(boxes, im_w, im_h)\n                    \n                if boxes.shape[0]>0:\n                    enboxes.append(boxes)\n                    enscores.append(scores)\n                \n        boxes, scores, labels = run_wbf(enboxes, enscores, image_size = im_w-1, iou_thr=0.55, skip_box_thr=0.2)\n        boxes = boxes.astype(np.int32).clip(min=0, max=1024)\n\n        boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n        boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n        boxes = boxes[scores >= 0.15].astype(np.int32)\n        scores = scores[scores >=float(0.15)]\n\n        result = {\n                    'image_id': image_id,\n                    'PredictionString': format_prediction_string(boxes, scores)\n                }\n        results.append(result)\n\n        if count<10:\n            sample = cv2.imread(os.path.join('../input/global-wheat-detection/test/', image_id+'.jpg'))\n            sample = cv2.cvtColor(sample, cv2.COLOR_RGB2BGR)\n            for box, score in zip(boxes,scores):\n                cv2.rectangle(sample,\n                          (box[0], box[1]),\n                          (box[2]+box[0], box[3]+box[1]),\n                          (220, 0, 0), 4)\n                cv2.putText(sample, '%.2f'%(score), (box[0], box[1]+box[3]), cv2.FONT_HERSHEY_SIMPLEX ,  \n                       0.7, (255,255,255), 2, cv2.LINE_AA)\n            ax[count%5][count//5].imshow(sample)\n            count+=1\n        \n    plt.show() \n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = detect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save detection results to submit","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\ntest_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\ntest_df.head()\ntest_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}