{"cells":[{"metadata":{},"cell_type":"markdown","source":"> This Kernal is continution of https://www.kaggle.com/ashoksrinivas/eda-pre-processing-and-anchor-generation-for-yolo"},{"metadata":{},"cell_type":"markdown","source":"I suggest you to check above mentioned kernal before going through this kernal :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\nimport os, cv2\nimport pandas as pd\nimport numpy as np\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_images[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LABELS = ['wheat']\n\nIMAGE_H, IMAGE_W = 416, 416\nGRID_H,  GRID_W  = 13 , 13\nBOX              = 5\nCLASS            = len(LABELS)\nCLASS_WEIGHTS    = np.ones(CLASS, dtype='float32')\nOBJ_THRESHOLD    = 0.3\nNMS_THRESHOLD    = 0.3\n\nNO_OBJECT_SCALE  = 1.0\nOBJECT_SCALE     = 5.0\nCOORD_SCALE      = 1.0\nCLASS_SCALE      = 1.0\nBATCH_SIZE       = 16\nWARM_UP_BATCHES  = 100\nTRUE_BOX_BUFFER  = 50","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check this https://www.kaggle.com/ashoksrinivas/eda-pre-processing-and-anchor-generation-for-yolo to understand how i extracted ANCHORS."},{"metadata":{"trusted":true},"cell_type":"code","source":"ANCHORS          = [0.67,0.67, 0.89,1.17, 1.23,0.70, 1.63,1.14, 1.69,2.06]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BoundBox:\n    def __init__(self, xmin, ymin, xmax, ymax, c = None, classes = None):\n        self.xmin = xmin\n        self.ymin = ymin\n        self.xmax = xmax\n        self.ymax = ymax\n        \n        self.c     = c\n        self.classes = classes\n\n        self.label = -1\n        self.score = -1\n\n    def get_label(self):\n        if self.label == -1:\n            self.label = np.argmax(self.classes)\n        \n        return self.label\n    \n    def get_score(self):\n        if self.score == -1:\n            self.score = self.classes[self.get_label()]\n            \n        return self.score\n\nclass WeightReader:\n    def __init__(self, weight_file):\n        self.offset = 4\n        self.all_weights = np.fromfile(weight_file, dtype='float32')\n        \n    def read_bytes(self, size):\n        self.offset = self.offset + size\n        return self.all_weights[self.offset-size:self.offset]\n    \n    def reset(self):\n        self.offset = 4\n\ndef bbox_iou(box1, box2):\n    intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n    intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])  \n    \n    intersect = intersect_w * intersect_h\n\n    w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n    w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n    \n    union = w1*h1 + w2*h2 - intersect\n    \n    return float(intersect) / union\n\ndef draw_boxes(image, boxes, labels):\n    image_h, image_w, _ = image.shape\n\n    for box in boxes:\n        xmin = int(box.xmin*image_w)\n        ymin = int(box.ymin*image_h)\n        xmax = int(box.xmax*image_w)\n        ymax = int(box.ymax*image_h)\n\n        cv2.rectangle(image, (xmin,ymin), (xmax,ymax), (0,255,0), 3)\n        cv2.putText(image, \n                    labels[box.get_label()] + ' ' + str(box.get_score()), \n                    (xmin, ymin - 13), \n                    cv2.FONT_HERSHEY_SIMPLEX, \n                    1e-3 * image_h, \n                    (0,255,0), 2)\n        \n    return image          \n        \ndef decode_netout(netout, anchors, nb_class, obj_threshold, nms_threshold):\n    grid_h, grid_w, nb_box = netout.shape[:3]\n\n    boxes = []\n    \n    # decode the output by the network\n    netout[..., 4]  = _sigmoid(netout[..., 4])\n    netout[..., 5:] = netout[..., 4][..., np.newaxis] * _softmax(netout[..., 5:])\n    netout[..., 5:] *= netout[..., 5:] > obj_threshold\n    \n    for row in range(grid_h):\n        for col in range(grid_w):\n            for b in range(nb_box):\n                # from 4th element onwards are confidence and class classes\n                classes = netout[row,col,b,5:]\n                \n                if np.sum(classes) > 0:\n                    # first 4 elements are x, y, w, and h\n                    x, y, w, h = netout[row,col,b,:4]\n\n                    x = (col + _sigmoid(x)) / grid_w # center position, unit: image width\n                    y = (row + _sigmoid(y)) / grid_h # center position, unit: image height\n                    w = anchors[2 * b + 0] * np.exp(w) / grid_w # unit: image width\n                    h = anchors[2 * b + 1] * np.exp(h) / grid_h # unit: image height\n                    confidence = netout[row,col,b,4]\n                    \n                    box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, confidence, classes)\n                    \n                    boxes.append(box) \n\n   \n    # suppress non-maximal boxes\n    for c in range(nb_class):\n        sorted_indices = list(reversed(np.argsort([box.classes[c] for box in boxes])))\n\n        for i in range(len(sorted_indices)):\n            index_i = sorted_indices[i]\n            \n            if boxes[index_i].classes[c] == 0: \n                continue\n            else:\n                for j in range(i+1, len(sorted_indices)):\n                    index_j = sorted_indices[j]\n                    \n                    if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_threshold:\n                        boxes[index_j].classes[c] = 0\n                        \n    # remove the boxes which are less likely than a obj_threshold\n    boxes = [box for box in boxes if box.get_score() > obj_threshold]\n    \n    return boxes    \n\ndef compute_overlap(a, b):\n    \"\"\"\n    Code originally from https://github.com/rbgirshick/py-faster-rcnn.\n    Parameters\n    ----------\n    a: (N, 4) ndarray of float\n    b: (K, 4) ndarray of float\n    Returns\n    -------\n    overlaps: (N, K) ndarray of overlap between boxes and query_boxes\n    \"\"\"\n    area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])\n\n    iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0])\n    ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1])\n\n    iw = np.maximum(iw, 0)\n    ih = np.maximum(ih, 0)\n\n    ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih\n\n    ua = np.maximum(ua, np.finfo(float).eps)\n\n    intersection = iw * ih\n\n    return intersection / ua  \n    \ndef compute_ap(recall, precision):\n    \"\"\" Compute the average precision, given the recall and precision curves.\n    Code originally from https://github.com/rbgirshick/py-faster-rcnn.\n    # Arguments\n        recall:    The recall curve (list).\n        precision: The precision curve (list).\n    # Returns\n        The average precision as computed in py-faster-rcnn.\n    \"\"\"\n    # correct AP calculation\n    # first append sentinel values at the end\n    mrec = np.concatenate(([0.], recall, [1.]))\n    mpre = np.concatenate(([0.], precision, [0.]))\n\n    # compute the precision envelope\n    for i in range(mpre.size - 1, 0, -1):\n        mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n    # to calculate area under PR curve, look for points\n    # where X axis (recall) changes value\n    i = np.where(mrec[1:] != mrec[:-1])[0]\n\n    # and sum (\\Delta recall) * prec\n    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n    return ap      \n        \ndef _interval_overlap(interval_a, interval_b):\n    x1, x2 = interval_a\n    x3, x4 = interval_b\n\n    if x3 < x1:\n        if x4 < x1:\n            return 0\n        else:\n            return min(x2,x4) - x1\n    else:\n        if x2 < x3:\n             return 0\n        else:\n            return min(x2,x4) - x3          \n\ndef _sigmoid(x):\n    return 1. / (1. + np.exp(-x))\n\ndef _softmax(x, axis=-1, t=-100.):\n    x = x - np.max(x)\n    \n    if np.min(x) < t:\n        x = x/np.min(x)*t\n        \n    e_x = np.exp(x)\n    \n    return e_x / e_x.sum(axis, keepdims=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly reffer this blog before going to BatchGenerator..."},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize(image):\n    return image/255.\n\n#pre-processing\nfrom keras.utils import Sequence\nimport copy\nclass BatchGenerator(Sequence):\n    def __init__(self, images, \n                       config, \n                       shuffle=True, \n                       jitter=True, \n                       norm=None):\n        self.generator = None\n\n        self.images = images\n        self.config = config\n\n        self.shuffle = shuffle\n        self.jitter  = jitter\n        self.norm    = norm\n\n        self.anchors = [BoundBox(0, 0, config['ANCHORS'][2*i], config['ANCHORS'][2*i+1]) for i in range(int(len(config['ANCHORS'])//2))]\n\n        ### augmentors by https://github.com/aleju/imgaug\n        sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n\n        # Define our sequence of augmentation steps that will be applied to every image\n        # All augmenters with per_channel=0.5 will sample one value _per image_\n        # in 50% of all cases. In all other cases they will sample new values\n        # _per channel_.\n        self.aug_pipe = iaa.Sequential(\n            [\n                # apply the following augmenters to most images\n                #iaa.Fliplr(0.5), # horizontally flip 50% of all images\n                #iaa.Flipud(0.2), # vertically flip 20% of all images\n                #sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width\n                sometimes(iaa.Affine(\n                    #scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis\n                    #translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)\n                    #rotate=(-5, 5), # rotate by -45 to +45 degrees\n                    #shear=(-5, 5), # shear by -16 to +16 degrees\n                    #order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                    #cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                    #mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n                )),\n                # execute 0 to 5 of the following (less important) augmenters per image\n                # don't execute all of them, as that would often be way too strong\n                iaa.SomeOf((0, 5),\n                    [\n                        #sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n                        iaa.OneOf([\n                            iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0\n                            iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7\n                            iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7\n                        ]),\n                        iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images\n                        #iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                        # search either for all edges or for directed edges\n                        #sometimes(iaa.OneOf([\n                        #    iaa.EdgeDetect(alpha=(0, 0.7)),\n                        #    iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),\n                        #])),\n                        iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images\n                        iaa.OneOf([\n                            iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels\n                            #iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),\n                        ]),\n                        #iaa.Invert(0.05, per_channel=True), # invert color channels\n                        iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n                        iaa.Multiply((0.5, 1.5), per_channel=0.5), # change brightness of images (50-150% of original value)\n                        iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast\n                        #iaa.Grayscale(alpha=(0.0, 1.0)),\n                        #sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n                        #sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around\n                    ],\n                    random_order=True\n                )\n            ],\n            random_order=True\n        )\n\n        if shuffle: np.random.shuffle(self.images)\n\n    def __len__(self):\n        return int(np.ceil(float(len(self.images))/self.config['BATCH_SIZE']))   \n\n    def num_classes(self):\n        return len(self.config['LABELS'])\n\n    def size(self):\n        return len(self.images)    \n\n    def load_annotation(self, i):\n        annots = []\n\n        for obj in self.images[i]['object']:\n            annot = [obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], self.config['LABELS'].index(obj['name'])]\n            annots += [annot]\n\n        if len(annots) == 0: annots = [[]]\n\n        return np.array(annots)\n\n    def load_image(self, i):\n        return cv2.imread(self.images[i]['filename'])\n\n    def __getitem__(self, idx):\n        l_bound = idx*self.config['BATCH_SIZE']\n        r_bound = (idx+1)*self.config['BATCH_SIZE']\n\n        if r_bound > len(self.images):\n            r_bound = len(self.images)\n            l_bound = r_bound - self.config['BATCH_SIZE']\n\n        instance_count = 0\n\n        x_batch = np.zeros((r_bound - l_bound, self.config['IMAGE_H'], self.config['IMAGE_W'], 3))                         # input images\n        b_batch = np.zeros((r_bound - l_bound, 1     , 1     , 1    ,  self.config['TRUE_BOX_BUFFER'], 4))   # list of self.config['TRUE_self.config['BOX']_BUFFER'] GT boxes\n        y_batch = np.zeros((r_bound - l_bound, self.config['GRID_H'],  self.config['GRID_W'], self.config['BOX'], 4+1+len(self.config['LABELS'])))                # desired network output\n\n        for train_instance in self.images[l_bound:r_bound]:\n            # augment input image and fix object's position and size\n            img, all_objs = self.aug_image(train_instance, jitter=self.jitter)\n            \n            # construct output from object's x, y, w, h\n            true_box_index = 0\n            \n            for obj in all_objs:\n                if obj['xmax'] > obj['xmin'] and obj['ymax'] > obj['ymin'] and obj['name'] in self.config['LABELS']:\n                    center_x = .5*(obj['xmin'] + obj['xmax'])\n                    center_x = center_x / (float(self.config['IMAGE_W']) / self.config['GRID_W'])\n                    center_y = .5*(obj['ymin'] + obj['ymax'])\n                    center_y = center_y / (float(self.config['IMAGE_H']) / self.config['GRID_H'])\n\n                    grid_x = int(np.floor(center_x))\n                    grid_y = int(np.floor(center_y))\n\n                    if grid_x < self.config['GRID_W'] and grid_y < self.config['GRID_H']:\n                        obj_indx  = self.config['LABELS'].index(obj['name'])\n                        \n                        center_w = (obj['xmax'] - obj['xmin']) / (float(self.config['IMAGE_W']) / self.config['GRID_W']) # unit: grid cell\n                        center_h = (obj['ymax'] - obj['ymin']) / (float(self.config['IMAGE_H']) / self.config['GRID_H']) # unit: grid cell\n                        \n                        box = [center_x, center_y, center_w, center_h]\n\n                        # find the anchor that best predicts this box\n                        best_anchor = -1\n                        max_iou     = -1\n                        \n                        shifted_box = BoundBox(0, \n                                               0,\n                                               center_w,                                                \n                                               center_h)\n                        \n                        for i in range(len(self.anchors)):\n                            anchor = self.anchors[i]\n                            iou    = bbox_iou(shifted_box, anchor)\n                            \n                            if max_iou < iou:\n                                best_anchor = i\n                                max_iou     = iou\n                                \n                        # assign ground truth x, y, w, h, confidence and class probs to y_batch\n                        y_batch[instance_count, grid_y, grid_x, best_anchor, 0:4] = box\n                        y_batch[instance_count, grid_y, grid_x, best_anchor, 4  ] = 1.\n                        y_batch[instance_count, grid_y, grid_x, best_anchor, 5+obj_indx] = 1\n                        \n                        # assign the true box to b_batch\n                        b_batch[instance_count, 0, 0, 0, true_box_index] = box\n                        \n                        true_box_index += 1\n                        true_box_index = true_box_index % self.config['TRUE_BOX_BUFFER']\n                            \n            # assign input image to x_batch\n            if self.norm != None: \n                x_batch[instance_count] = self.norm(img)\n            else:\n                # plot image and bounding boxes for sanity check\n                for obj in all_objs:\n                    if obj['xmax'] > obj['xmin'] and obj['ymax'] > obj['ymin']:\n                        cv2.rectangle(img[:,:,::-1], (obj['xmin'],obj['ymin']), (obj['xmax'],obj['ymax']), (255,0,0), 3)\n                        cv2.putText(img[:,:,::-1], obj['name'], \n                                    (obj['xmin']+2, obj['ymin']+12), \n                                    0, 1.2e-3 * img.shape[0], \n                                    (0,255,0), 2)\n                        \n                x_batch[instance_count] = img\n\n            # increase instance counter in current batch\n            instance_count += 1  \n\n        #print(' new batch created', idx)\n\n        return [x_batch, b_batch], y_batch\n\n    def on_epoch_end(self):\n        if self.shuffle: np.random.shuffle(self.images)\n\n    def aug_image(self, train_instance, jitter):\n        image_name = train_instance['filename']\n        image = cv2.imread(image_name)\n\n        if image is None: print('Cannot find ', image_name)\n\n        h, w, c = image.shape\n        all_objs = copy.deepcopy(train_instance['object'])\n\n        if jitter:\n            ### scale the image\n            scale = np.random.uniform() / 10. + 1.\n            image = cv2.resize(image, (0,0), fx = scale, fy = scale)\n\n            ### translate the image\n            max_offx = (scale-1.) * w\n            max_offy = (scale-1.) * h\n            offx = int(np.random.uniform() * max_offx)\n            offy = int(np.random.uniform() * max_offy)\n            \n            image = image[offy : (offy + h), offx : (offx + w)]\n\n            ### flip the image\n            flip = np.random.binomial(1, .5)\n            if flip > 0.5: image = cv2.flip(image, 1)\n                \n            image = self.aug_pipe.augment_image(image)            \n            \n        # resize the image to standard size\n        image = cv2.resize(image, (self.config['IMAGE_H'], self.config['IMAGE_W']))\n        image = image[:,:,::-1]\n\n        # fix object's position and size\n        for obj in all_objs:\n            for attr in ['xmin', 'xmax']:\n                if jitter: obj[attr] = int(obj[attr] * scale - offx)\n                    \n                obj[attr] = int(obj[attr] * float(self.config['IMAGE_W']) / w)\n                obj[attr] = max(min(obj[attr], self.config['IMAGE_W']), 0)\n                \n            for attr in ['ymin', 'ymax']:\n                if jitter: obj[attr] = int(obj[attr] * scale - offy)\n                    \n                obj[attr] = int(obj[attr] * float(self.config['IMAGE_H']) / h)\n                obj[attr] = max(min(obj[attr], self.config['IMAGE_H']), 0)\n\n            if jitter and flip > 0.5:\n                xmin = obj['xmin']\n                obj['xmin'] = self.config['IMAGE_W'] - obj['xmax']\n                obj['xmax'] = self.config['IMAGE_W'] - xmin\n                \n        return image, all_objs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generator_config = {\n    'IMAGE_H'         : IMAGE_H, \n    'IMAGE_W'         : IMAGE_W,\n    'GRID_H'          : GRID_H,  \n    'GRID_W'          : GRID_W,\n    'BOX'             : BOX,\n    'LABELS'          : LABELS,\n    'CLASS'           : len(LABELS),\n    'ANCHORS'         : ANCHORS,\n    'BATCH_SIZE'      : BATCH_SIZE,\n    'TRUE_BOX_BUFFER' : 50,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_valid_split = int(0.8*len(all_images))\n\ntrain_batch = BatchGenerator(all_images[:train_valid_split], generator_config)\nvalid_batch = BatchGenerator(all_images[train_valid_split:], generator_config, norm=normalize)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image0 = train_batch[1][0][0][0]\nimage1 = train_batch[2][0][0][0]\nimage2 = train_batch[3][0][0][0]\nplt.imshow(image0.astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(image1.astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(image2.astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}