{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Fastai v1 : Object detection Tutorial","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### If you like, please don't forget to upvote","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install object-detection-fastai","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom collections import defaultdict\nimport os\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import *\nimport seaborn as sns \nimport matplotlib.pyplot as plt\nimport matplotlib.image as immg\nfrom sklearn.model_selection import StratifiedKFold,KFold\n\nfrom object_detection_fastai.helper.object_detection_helper import *\nfrom object_detection_fastai.loss.RetinaNetFocalLoss import RetinaNetFocalLoss\nfrom object_detection_fastai.models.RetinaNet import RetinaNet\nfrom object_detection_fastai.callbacks.callbacks import BBLossMetrics, BBMetrics, PascalVOCMetric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set_style('darkgrid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/global-wheat-detection');path.ls()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(path/'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = train.image_id.value_counts()\ntr = pd.DataFrame({'image_id':tr.index,'wheat_count':tr.values})\ntr = tr.sample(frac=1.,random_state=2020).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fnames = get_files(path/'train')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DataFrame Format:","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"* A function needs to return the coordinates then the labels. \n* Let's look at an example quickly; from fastai before we preprocess the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"coco_source = untar_data(URLs.COCO_TINY)\nimages, lbl_bbox = get_annotations(coco_source/'train.json')\nimg2bbox = dict(zip(images, lbl_bbox))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images[0],lbl_bbox[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* we have **images** and **labels**\n* Then both get **ziped in a dictionary**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## fastai images label format for object detection\n\n* Now we have our actual data frame above, we need to make some adjustments. \n\n* Currently our coordinates are **x, y , w , h** and we want **x1, y1, x2, y2**\n\n* To convert it, we need to add our width and height to the respective x and y. \n\n* To achieve x2, y2 those we simply add **width to x** and **height to y** :---- **x2 = x + w** and **y2 = y + h**\n\n\n\n* **dict contains image_names, bbox, label**\n\n**format**\n\n**{ 'dgdsf244.jpg' : [ [ [ bbox1 ][ bbox2] ] , \n                        [ 'label1' ,'label2' ] ]**\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lbl_img(train):\n    wheat2bbox = {}\n    train['label'] = 'wheat'\n    grp = train.image_id.unique()\n    tr_gr = train.groupby(['image_id'])\n    from tqdm.notebook import tqdm\n    for i in tqdm(range(len(grp))):\n        name = str(grp[i]) + '.jpg'\n        bbox = []\n        temp_b = []\n        temp = tr_gr.get_group(grp[i])\n        tt = temp.loc[:,'bbox'].values\n        for j in range(len(temp)):\n            t = tt[j][1:-1].split(',')\n            t = [float(x) for x in t]  # x,y, width, height\n            # Currently our coordinates are x,y,w,h and we want x1,y1,x2,y2\n            # To convert it, we need to add our width and height to the respective x and y.\n            t[2],t[3] = t[0]+t[2],t[1]+t[3]  \n            # To achieve x2,y2 those we simply add width to x and height to y :\n            # x2 = x + w and y2 = y + h\n            t1 = [t[1],t[0],t[3],t[2]]   # inverse in fromat w,h for fastai\n            temp_b.append(t1)\n        bbox.append(temp_b)\n        bbox.append(['wheat']*len(temp))\n        wheat2bbox[name] = bbox\n    return wheat2bbox","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wheat2bbox = get_lbl_img(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Let's look at the dict of images and labels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a = wheat2bbox['d1f73158a.jpg'];a","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Given a path below function will simply return its bbox and labels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"get_y_func = lambda o: wheat2bbox[Path(o).name] ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now lets create a Object detection data block","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = pd.read_csv('../input/global-wheat-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = sample.image_id.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DataBunch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ts = (ObjectItemList.from_df(sample,path, folder = 'test' , suffix = '.jpg',cols='image_id'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (ObjectItemList.from_df(tr,path, folder = 'train' , suffix = '.jpg',cols='image_id')\n        #Where are the images? ->\n        .split_by_rand_pct(0.2)                          \n        #How to split in train/valid? -> randomly with the default 20% in valid\n        .label_from_func(get_y_func)\n        #How to find the labels? -> use get_y_func on the file name of the data\n        .transform(size=512)\n        .add_test(ts)\n        #Data augmentation? -> Standard transforms; also transform the label images\n        .databunch(bs=2, collate_fn=bb_pad_collate))   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(1 , figsize = (8,8) ,ds_type=DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data.train_ds),len(data.valid_ds),len(data.test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"size = 512","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### What Is an Anchor Box?\n* **Anchor boxes are a set of predefined bounding boxes of a certain height and width. These boxes are defined to capture the scale and aspect ratio of specific object classes you want to detect** and are typically chosen based on object sizes in your training datasets. During detection, the predefined anchor boxes are tiled across the image. The network predicts the probability and other attributes, such as background, intersection over union (IoU) and offsets for every tiled anchor box. The predictions are used to refine each individual anchor box. You can define several anchor boxes, each for a different object size. Anchor boxes are fixed initial boundary box guesses.\n\n* **The network does not directly predict bounding boxes, but rather predicts the probabilities and refinements that correspond to the tiled anchor boxes.** The network returns a unique set of predictions for every anchor box defined. The final feature map represents object detections for each class. The use of anchor boxes enables a network to detect multiple objects, objects of different scales, and overlapping objects.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Advantage of Using Anchor Boxes\n* When using anchor boxes, you can evaluate all object predictions at once. Anchor boxes eliminate the need to scan an image with a sliding window that computes a separate prediction at every potential position. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"anchors = create_anchors(sizes=[(32,32),(16,16),(8,8),(4,4)], ratios=[0.5, 1, 2], scales=[0.35, 0.55, 0.75, 1, 1.25, 1.45])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(10,10))\nax.imshow(image2np(data.valid_ds[0][0].data))\n\nfor i, bbox in enumerate(anchors[:18]):\n    bb = bbox.numpy()\n    x = (bb[0] + 1) * size / 2 \n    y = (bb[1] + 1) * size / 2 \n    w = bb[2] * size / 2\n    h = bb[3] * size / 2\n    \n    rect = [x,y,w,h]\n    draw_rect(ax,rect)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(anchors)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_classes = data.train_ds.c\n\ncrit = RetinaNetFocalLoss(anchors)\n\nencoder = create_body(models.resnet18, True, -2)\n\nmodel = RetinaNet(encoder, n_classes=data.train_ds.c, n_anchors=18, sizes=[32,16,8,4], chs=32, final_bias = -4., n_conv = 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* **n_anchors = len(ratios) x len(scales)**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"voc = PascalVOCMetric(anchors, size, [i for i in data.train_ds.y.classes[1:]])\nlearn = Learner(data,\n                model, \n                loss_func=crit,\n                callback_fns=[BBMetrics],\n                metrics=[voc],\n                model_dir = '/kaggle/working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.split([model.encoder[6], model.c5top5]);\nlearn.freeze_to(-2)\n#learn = learn.to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.lr_find()\n#learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.unfreeze()\nlearn.fit_one_cycle(4, 1e-3 ,callbacks = [SaveModelCallback(learn, every ='improvement', monitor ='AP-wheat', name ='best_wheat')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('best_wheat');\nlearn.export('/kaggle/working/gwheat.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_results_side_by_side(learn, anchors, detect_thresh=0.5, nms_thresh=0.1, image_count=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## If you reached till here please don't forget to upvote.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prediction Test images helpful functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_output(item,bboxs_tot,scores_tot):\n    fig,ax = plt.subplots(figsize=(10,10))\n    ax.imshow(image2np(item.data))\n    plt.axis('off')\n    area_max = 512**2/5 \n    for bbox, c in zip(bboxs_tot[0], scores_tot[0].numpy()):\n        txt = 'wheat, {0:.4f}'.format(c)\n        if bbox[2]*bbox[3] <= area_max:\n            draw_rect(ax, [bbox[1],bbox[0],bbox[3],bbox[2]], text=txt,text_size=12,color='red')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_preds_show(item,clas,bboxs,show_img,cnt,i):\n    detect_thresh=0.4   # set your own detection threshold\n    nms_thresh=0.1\n    pred_string = []\n    scores_tot = []\n    bboxs_tot = []\n    show_img = True if i<cnt else False\n    for clas_pred, bbox_pred in list(zip(clas, bboxs)):\n        bbox_pred, scores, preds = process_output(clas_pred, bbox_pred, anchors, detect_thresh)\n        if bbox_pred is not None:\n            to_keep = nms(bbox_pred, scores, nms_thresh)\n            bbox_pred, preds, scores = bbox_pred[to_keep].cpu(), preds[to_keep].cpu(), scores[to_keep].cpu()\n        t_sz = torch.Tensor([size])[None].cpu()\n        if bbox_pred is not None:\n            bbox_pred = to_np(rescale_boxes(bbox_pred, t_sz))\n                # change from center to top left\n            bbox_pred[:, :2] = bbox_pred[:, :2] - bbox_pred[:, 2:] / 2\n            bboxs_tot.append(bbox_pred)\n            scores_tot.append(scores)\n    if show_img:\n        show_output(item,bboxs_tot,scores_tot)\n    area_max = (1024**2)/5\n    for s,bbx in zip(scores_tot[0].numpy(),bboxs_tot[0]):\n        bbx = [int(round(x)) for x in bbx*2]\n        if bbx[2]*bbx[3] <= area_max :\n            res = \"{0:.4f} {1} {2} {3} {4}\".format(s,bbx[1],bbx[0],bbx[3],bbx[2])\n            pred_string.append(res)\n    return pred_string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prediction(show_img=True,cnt=10): \n    # Set show img True to see img or else false for bboxs only, cnt for number of images to show\n    preds_str = {}\n    for i in range(len(data.test_ds)):\n        item = learn.data.test_ds[i][0]  #Pick one image\n        batch = learn.data.one_item(item)\n        clas,bboxs,xtr = learn.pred_batch(batch=batch)\n        prd = process_preds_show(item,clas,bboxs,show_img,cnt,i) \n        preds_str[image_id[i]] = \" \".join(prd)\n    return preds_str","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = get_prediction()\n# Set False to not show images\n# Regardless of that it will give prediction string ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.DataFrame.from_dict(prediction,orient='index').reset_index()\nsubmit.columns = ['image_id','PredictionString']\nsubmit.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}