{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{"height":"calc(100% - 180px)","left":"10px","top":"150px","width":"279.273px"},"toc_section_display":true,"toc_window_display":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nfrom utils import *\n\n%load_ext autoreload","metadata":{"ExecuteTime":{"end_time":"2019-08-26T02:59:53.861942Z","start_time":"2019-08-26T02:59:46.436125Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Extraction","metadata":{}},{"cell_type":"code","source":"image = torch.zeros((1, 3, 800, 800)).float()\nimage_size = (800, 800)\n\n# bbox -> y1, x1, y2, x2\nbbox = torch.FloatTensor([[20, 30, 400, 500], [300, 400, 500, 600]])\nlabels = torch.LongTensor([6, 8])\n\nsub_sample = 16","metadata":{"ExecuteTime":{"end_time":"2019-08-26T02:59:53.900978Z","start_time":"2019-08-26T02:59:53.863898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vgg16 = torchvision.models.vgg16(pretrained=True)\nreq_features = vgg16.features[:30]\nprint(req_features)\noutput_map = req_features(image)\nprint(output_map.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.620894Z","start_time":"2019-08-26T02:59:53.907961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target Anchor Box","metadata":{}},{"cell_type":"markdown","source":"## Assign template anchor boxes with origin (0, 0)\n\nWe will use `anchor_scales(feature map)` of **8, 16, 32**, `ratio` of **0.5, 1, 2** and `sub sampling` of **16** (Since we have pooled our image from 800 px to 50px). Now every pixel in the output feature map maps to corresponding 16 * 16 pixels in the image.\n\nanchor_template = (9, 4)","metadata":{}},{"cell_type":"code","source":"anchor_scale = [8, 16, 32]\nratio = [0.5, 1, 2] # H/W\n\nlen_anchor_scale = len(anchor_scale)\nlen_ratio = len(ratio)\nlen_anchor_template = len_anchor_scale * len_ratio\nanchor_template = np.zeros((9, 4))\n\nfor idx, scale in enumerate(anchor_scale):\n    h = scale * np.sqrt(ratio) * sub_sample\n    w = scale / np.sqrt(ratio) * sub_sample\n    y1 = -h/2\n    x1 = -w/2\n    y2 = h/2\n    x2 = w/2\n    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 0] = y1\n    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 1] = x1\n    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 2] = y2\n    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 3] = x2\n\nprint(anchor_template)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.857767Z","start_time":"2019-08-26T03:00:14.694669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.sqrt(ratio)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.873755Z","start_time":"2019-08-26T03:00:14.859796Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Generate anchor at all the feature map location","metadata":{}},{"cell_type":"markdown","source":"### Generate center coors for all the feature map pixels\nctr -> (50, 50, 2)","metadata":{}},{"cell_type":"code","source":"feature_map_size = (50, 50)\n# The first center coors is (8, 8)\nctr_y = np.arange(8, 800, 16)\nctr_x = np.arange(8, 800, 16)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.879709Z","start_time":"2019-08-26T03:00:14.875752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ctr = np.zeros((*feature_map_size, 2))\nfor idx, y in enumerate(ctr_y):\n    ctr[idx, :, 0] = y\n    ctr[idx, :, 1] = ctr_x\nprint(ctr.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.891685Z","start_time":"2019-08-26T03:00:14.881704Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Add center coors and previous anchor boxes coors\n\nanchors -> (50, 50, 9, 4)","metadata":{}},{"cell_type":"code","source":"anchors = np.zeros((*feature_map_size, 9, 4))\n\nfor idx_y in range(feature_map_size[0]):\n    for idx_x in range(feature_map_size[1]):\n        anchors[idx_y, idx_x] = (ctr[idx_y, idx_x] + anchor_template.reshape(-1, 2, 2)).reshape(-1, 4)\nprint(anchors.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.922592Z","start_time":"2019-08-26T03:00:14.895665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"anchors = anchors.reshape(-1, 4)\nprint(anchors.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.93157Z","start_time":"2019-08-26T03:00:14.925584Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Assign labels and location of objects to each and every anchor","metadata":{}},{"cell_type":"markdown","source":"### Find the index of all valid anchor boxes\n\nvalid anchor boxes -> inside the image","metadata":{}},{"cell_type":"code","source":"valid_index = np.where((anchors[:, 0] >= 0)\n                      &(anchors[:, 1] >= 0)\n                      &(anchors[:, 2] <= 800)\n                      &(anchors[:, 3] <= 800))[0]\nprint(valid_index.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.945542Z","start_time":"2019-08-26T03:00:14.93357Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We assign a **positive label to two kind of anchors** a) The anchor/anchors with the highest Intersection-over-Union(IoU) overlap with a ground-truth-box or b) An anchor that has an IoU overlap higher than 0.7 with ground-truth box.\n\n\n**Note that single ground-truth object may assign positive labels to multiple anchors.**\n\n\nc) We assign a **negative label** to a non-positive anchor if its IoU ratio is lower than 0.3 for all ground-truth boxes. \n\nd) Anchors that are **neither positive nor negitive** do not contribute to the training objective.","metadata":{}},{"cell_type":"markdown","source":"### create an empty label array with inside_index shape and fill with -1. ","metadata":{}},{"cell_type":"code","source":"valid_labels = np.empty((valid_index.shape[0],), dtype=np.int32)\nvalid_labels.fill(-1)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.952514Z","start_time":"2019-08-26T03:00:14.947536Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create valid anchors","metadata":{}},{"cell_type":"code","source":"valid_anchors = anchors[valid_index]","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.958498Z","start_time":"2019-08-26T03:00:14.95451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(valid_anchors.shape)\nprint(bbox.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:14.969473Z","start_time":"2019-08-26T03:00:14.962512Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Generate ious matrix between valid_anchors and bbox","metadata":{}},{"cell_type":"code","source":"ious = bbox_iou(valid_anchors, bbox.numpy())","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.293603Z","start_time":"2019-08-26T03:00:14.972458Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Label pos and neg anchors\n\nConsidering the scenarios of a and b, we need to find two things here\n- the highest iou for each gt_box and its corresponding anchor box\n- the highest iou for each anchor box and its corresponding ground truth box","metadata":{}},{"cell_type":"markdown","source":"#### Set `pos_iou_thres` and `neg_iou_thres` to set anchors","metadata":{}},{"cell_type":"code","source":"pos_iou_thres = 0.7\nneg_iou_thred = 0.3\n\nanchor_max_iou = np.amax(ious, axis=1)\npos_iou_anchor_label = np.where(anchor_max_iou >= pos_iou_thres)[0]\nneg_iou_anchor_label = np.where(anchor_max_iou < neg_iou_thred)[0]\nvalid_labels[pos_iou_anchor_label] = 1\nvalid_labels[neg_iou_anchor_label] = 0","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.304572Z","start_time":"2019-08-26T03:00:15.296592Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Assign labels to the anchors with highest iou with box","metadata":{}},{"cell_type":"code","source":"gt_max_iou = np.amax(ious, axis=0)\ngt_max_iou_anchor_label = np.where(ious == gt_max_iou)[0]\nprint(gt_max_iou_anchor_label)\nvalid_labels[gt_max_iou_anchor_label] = 1","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.314545Z","start_time":"2019-08-26T03:00:15.306566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sample positive and negtive anchors\n\n> we randomly sample 256 anchors in an image to compute the loss function of a mini-batch, where the sampled positive and negative anchors have a ratio of up to 1:1. If there are fewer than 128 positive samples in an image, we pad the mini-batch with negitive ones.","metadata":{}},{"cell_type":"code","source":"n_sample_anchors = 256\npos_ratio = 0.5\n\ntotal_n_pos = len(np.where(valid_labels == 1)[0])\nn_pos_sample = n_sample_anchors*pos_ratio if total_n_pos > n_sample_anchors*pos_ratio else total_n_pos\nn_neg_sample = n_sample_anchors - n_pos_sample","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.329506Z","start_time":"2019-08-26T03:00:15.321528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pos_index = np.where(valid_labels == 1)[0]\nif len(pos_index) > n_sample_anchors*pos_ratio:\n    disable_index = np.random.choice(pos_index, size=len(pos_index)-n_pos_sample, replace=False)\n    valid_labels[disable_index] = -1\n\nneg_index = np.where(valid_labels == 0)[0]\ndisable_index = np.random.choice(neg_index, size=len(neg_index) - n_neg_sample, replace=False)\nvalid_labels[disable_index] = -1","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.40231Z","start_time":"2019-08-26T03:00:15.335492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Assigning locations to anchor boxes\n\n```\nt_{x} = (x - x_{a})/w_{a}\nt_{y} = (y - y_{a})/h_{a}\nt_{w} = log(w/ w_a)\nt_{h} = log(h/ h_a)\n```\n**x, y , w, h** are the groud truth box center co-ordinates which has maxmimum iou with corresponding anchor, width and height. **x_a, y_a, h_a and w_a** and anchor boxes center cooridinates, width and height.","metadata":{}},{"cell_type":"code","source":"# Each anchor corresponds to a box\n\nargmax_iou = np.argmax(ious, axis=1)\nmax_iou_box = bbox[argmax_iou].numpy()\nprint(max_iou_box.shape)\nprint(valid_anchors.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.663612Z","start_time":"2019-08-26T03:00:15.404305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"anchor_loc_format_target = format_loc(valid_anchors, max_iou_box)\nprint(anchor_loc_format_target.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.673587Z","start_time":"2019-08-26T03:00:15.665606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Final labels and locations","metadata":{}},{"cell_type":"code","source":"anchor_target_labels = np.empty((len(anchors),), dtype=np.int32)\nanchor_target_format_locations = np.zeros((len(anchors), 4), dtype=np.float32)\n\nanchor_target_labels.fill(-1)\nanchor_target_labels[valid_index] = valid_labels\n\nanchor_target_format_locations[valid_index] = anchor_loc_format_target\n\nprint(anchor_target_labels.shape)\nprint(anchor_target_format_locations.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.683558Z","start_time":"2019-08-26T03:00:15.67558Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Region Proposal Network\n\nTo generate region proposals, we **slide a small network over the convolutional feature map output** that we obtained in the feature extraction module. This small network takes as input an n x n spatial window of the input convolutional feature map. Each sliding window is mapped to a lower-dimensional feature [512 features]. **This feature is fed into two sibling fully connected layers**\n- A box regrression layer\n- A box classification layer","metadata":{}},{"cell_type":"code","source":"mid_channel = 512\nin_channel = 512\nn_anchor = 9\n\nconv1 = nn.Conv2d(in_channel, mid_channel, 3, 1, 1)\nreg_layer = nn.Conv2d(mid_channel, n_anchor*4, 1, 1, 0)\ncls_layer = nn.Conv2d(mid_channel, n_anchor*2, 1, 1, 0)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:15.929932Z","start_time":"2019-08-26T03:00:15.685555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Initialize weights and bias\nThe paper tells that they initialized these layers with zero mean and 0.01 standard deviation for weights and zeros for base.","metadata":{}},{"cell_type":"code","source":"conv1.weight.data.normal_(0, 0.01)\nconv1.bias.data.zero_()\n\nreg_layer.weight.data.normal_(0, 0.01)\nreg_layer.bias.data.zero_()\n\ncls_layer.weight.data.normal_(0, 0.01)\ncls_layer.bias.data.zero_()","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:16.494946Z","start_time":"2019-08-26T03:00:15.93192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = conv1(output_map)\nanchor_pred_format_locations = reg_layer(x)\nanchor_pred_scores = cls_layer(x)\n\nprint(anchor_pred_format_locations.shape)\nprint(anchor_pred_scores.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:16.740261Z","start_time":"2019-08-26T03:00:16.49691Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Reformat the output","metadata":{}},{"cell_type":"markdown","source":"Lets **reformat these a bit and make it align with our anchor targets we designed previously**. We will also find the objectness scores for each anchor box, as this is used to for proposal layer which we will discuss in the next section","metadata":{}},{"cell_type":"code","source":"anchor_pred_format_locations = anchor_pred_format_locations.permute(0, 2, 3, 1).contiguous().view(1, -1, 4)\nprint(anchor_pred_format_locations.shape)\n\nanchor_pred_scores = anchor_pred_scores.permute(0, 2, 3, 1).contiguous().view(1, -1, 2)\nprint(anchor_pred_scores.shape)\n\nobjectness_pred_scores = anchor_pred_scores[:, :, 1]\nprint(objectness_pred_scores.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:16.916788Z","start_time":"2019-08-26T03:00:16.742255Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## RPN Loss\n\n![](./RPN-loss.png)\n\nwhere `p_{i}` is the predicted class label and `p_{i}^*` is the actual class score. `t_{i}` and `t_{i}^*` are the predicted co-oridinates and actual co-ordinates. The ground-truth label `p_{i}^*` is 1 if the the anchor is positive and 0 if the anchor is negative. We will see how this is done in Pytorch.","metadata":{}},{"cell_type":"code","source":"print(anchor_target_labels.shape)\nprint(anchor_target_format_locations.shape)\nprint(anchor_pred_scores.shape)\nprint(anchor_pred_format_locations.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:16.927771Z","start_time":"2019-08-26T03:00:16.918783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gt_rpn_format_locs = torch.from_numpy(anchor_target_format_locations)\ngt_rpn_scores = torch.from_numpy(anchor_target_labels)\n\nrpn_format_locs = anchor_pred_format_locations[0]\nrpn_scores = anchor_pred_scores[0]","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:16.948713Z","start_time":"2019-08-26T03:00:16.931755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### cls loss\npred_cls_scores and anchor_labels are the predited objectness score and actual objectness score of the RPN network. We will use the following loss functions for Regression and classification respectively.\nFor classification we use cross-entropy loss\n![](./Cross_entropy-loss.png)\nCross Entropy Loss","metadata":{}},{"cell_type":"code","source":"rpn_cls_loss = F.cross_entropy(rpn_scores, gt_rpn_scores.long(), ignore_index=-1)\nprint(rpn_cls_loss)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.501103Z","start_time":"2019-08-26T03:00:16.950698Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### reg loss for positive target anchors\nFor Regression we use smooth L1 loss as defined in the Fast RCNN paper,\n![](Smooth-L1-loss.png)\nSmooth L1 Loss\nThey used L1 loss instead of L2 loss because the values of predicted regression head of RPN are not bounded. **Regression loss is also applied to the bounding boxes which have positive label**","metadata":{}},{"cell_type":"code","source":"mask = gt_rpn_scores > 0\nmask_target_format_locs = gt_rpn_format_locs[mask]\nmask_pred_format_locs = rpn_format_locs[mask]\n\nprint(mask_target_format_locs.shape)\nprint(mask_pred_format_locs.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.551099Z","start_time":"2019-08-26T03:00:17.502069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = torch.abs(mask_target_format_locs - mask_pred_format_locs)\nrpn_loc_loss = ((x<0.5).float()*(x**2)*0.5 + (x>0.5).float()*(x-0.5)).sum()\nprint(rpn_loc_loss)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.731524Z","start_time":"2019-08-26T03:00:17.553094Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### RPN total loss","metadata":{}},{"cell_type":"code","source":"rpn_lambda = 10\nN_reg = mask.float().sum()\n\nrpn_loss = rpn_cls_loss + rpn_lambda / N_reg * rpn_loc_loss\nprint(rpn_loss)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.765438Z","start_time":"2019-08-26T03:00:17.733517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Generating proposals to feed Fast R-CNN network\n\nThe proposal function will take the following parameters\n- Weather training_mode or testing mode\n- nms_thresh\n- n_train_pre_nms — number of bboxes before nms during training\n- n_train_post_nms — number of bboxes after nms during training\n- n_test_pre_nms — number of bboxes before nms during testing\n- n_test_post_nms — number of bboxes after nms during testing\n- min_size — minimum height of the object required to create a proposal.","metadata":{}},{"cell_type":"markdown","source":"The Faster R_CNN says, RPN proposals highly overlap with each other. To reduced redundancy, we adopt non-maximum supression (NMS) on the proposal regions based on their cls scores. We fix the IoU threshold for NMS at 0.7, which leaves us about 2000 proposal regions per image. After an ablation study, the authors show that NMS does not harm the ultimate detection accuracy, but substantially reduces the number of proposals. After NMS, we use the top-N ranked proposal regions for detection. In the following we training Fast R-CNN using 2000 RPN proposals. During testing they evaluate only 300 proposals, they have tested this with various numbers and obtained this.","metadata":{}},{"cell_type":"code","source":"nms_thresh = 0.7\nn_train_pre_nms = 12000\nn_train_post_nms = 2000\nn_test_pre_nms = 6000\nn_test_post_nms = 300\nmin_size = 16","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.772414Z","start_time":"2019-08-26T03:00:17.768426Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- convert the loc predictions from the rpn network to bbox [y1, x1, y2, x2] format.\n- clip the predicted boxes to the image\n- Remove predicted boxes with either height or width < threshold (min_size).\n- Sort all (proposal, score) pairs by score from highest to lowest.\n- Take top pre_nms_topN (e.g. 12000 while training and 300 while testing).\n- Apply nms threshold > 0.7\n- Take top pos_nms_topN (e.g. 2000 while training and 300 while testing)","metadata":{"ExecuteTime":{"end_time":"2019-08-25T19:35:00.042227Z","start_time":"2019-08-25T19:35:00.034248Z"}}},{"cell_type":"markdown","source":"## convert the loc predictions from the rpn net to [y1, x1, y2, x2] format\n","metadata":{}},{"cell_type":"code","source":"print(anchors.shape)\nprint(anchor_pred_format_locations.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.786376Z","start_time":"2019-08-26T03:00:17.776418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rois = deformat_loc(anchors=anchors, formatted_base_anchor=anchor_pred_format_locations[0].data.numpy())\nprint(rois.shape)\nprint(rois)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.802334Z","start_time":"2019-08-26T03:00:17.788372Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## clip the predicted boxes to the image","metadata":{}},{"cell_type":"code","source":"rois[:, 0:4:2] = np.clip(rois[:, 0:4:2], a_min=0, a_max=image_size[0])\nrois[:, 1:4:2] = np.clip(rois[:, 1:4:2], a_min=0, a_max=image_size[1])\nprint(rois)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.833253Z","start_time":"2019-08-26T03:00:17.804329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Remove predicted boxes with either height or width < threshold","metadata":{}},{"cell_type":"code","source":"h = rois[:, 2] - rois[:, 0]\nw = rois[:, 3] - rois[:, 1]\n\nvalid_index = np.where((h>min_size)&(w>min_size))[0]\nvalid_rois = rois[valid_index]\nvalid_scores = objectness_pred_scores[0][valid_index].data.numpy()\n\nprint(valid_rois.shape)\nprint(valid_scores.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.846217Z","start_time":"2019-08-26T03:00:17.835246Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## sort all pairs by score from highest to lowest","metadata":{}},{"cell_type":"code","source":"valid_score_order = valid_scores.ravel().argsort()[::-1]\nprint(valid_score_order)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.883118Z","start_time":"2019-08-26T03:00:17.84921Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Take top pre_nms_topN","metadata":{}},{"cell_type":"code","source":"pre_train_valid_score_order = valid_score_order[:n_train_pre_nms]\npre_train_valid_rois = valid_rois[pre_train_valid_score_order]\npre_train_valid_scores = valid_scores[pre_train_valid_score_order]\n\nprint(pre_train_valid_rois.shape)\nprint(pre_train_valid_scores.shape)\nprint(pre_train_valid_score_order.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:17.893098Z","start_time":"2019-08-26T03:00:17.885129Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Apply NMS threshold","metadata":{}},{"cell_type":"markdown","source":"- Take all the roi boxes [roi_array]\n- Find the areas of all the boxes [roi_area]\n- Take the indexes of order the probability score in descending order [order_array]\nkeep = []\nwhile order_array.size > 0:\n  - take the first element in order_array and append that to keep  \n  - Find the area with all other boxes\n  - Find the index of all the boxes which have high overlap with this box\n  - Remove them from order array\n  - Iterate this till we get the order_size to zero (while loop)\n- Ouput the keep variable which tells what indexes to consider.","metadata":{}},{"cell_type":"code","source":"keep_index = nms(rois=pre_train_valid_rois, scores=pre_train_valid_scores, nms_thresh=nms_thresh)\npost_train_valid_rois = pre_train_valid_rois[keep_index][:n_train_post_nms]\npost_train_valid_scores = pre_train_valid_scores[keep_index][:n_train_post_nms]\nprint(post_train_valid_rois.shape)\nprint(post_train_valid_scores.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:18.948273Z","start_time":"2019-08-26T03:00:17.896084Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Proposal targets","metadata":{}},{"cell_type":"markdown","source":"The Fast R-CNN network takes the region proposals (obtained from proposal layer in previous section), ground truth boxes and their respective labels as inputs. It will take the following parameters\n- n_sample: Number of samples to sample from roi, The default value is 128.\n- pos_ratio: the number of positive examples out of the n_samples. The default values is 0.25.\n- pos_iou_thesh: The minimum overlap of region proposal with any groundtruth object to consider it as positive label.\n- [neg_iou_threshold_lo, neg_iou_threshold_hi] : [0.0, 0.5], The overlap value bounding required to consider a region proposal as negitive [background object].","metadata":{}},{"cell_type":"markdown","source":"**This Step I think is used in the training process. In test step, we just use the proposal after NMS as the input of Fast RCNN network.**","metadata":{}},{"cell_type":"code","source":"n_sample = 128\npos_ratio = 0.25\npos_iou_thresh = 0.5\nneg_iou_thresh_hi = 0.5\nneg_iou_thresh_lo = 0.0","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:18.95625Z","start_time":"2019-08-26T03:00:18.950266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(post_train_valid_rois.shape)\nprint(post_train_valid_scores.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:18.965227Z","start_time":"2019-08-26T03:00:18.958245Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Find the iou of each ground truth object with the region proposals\n\nsame as 2.3.4","metadata":{}},{"cell_type":"code","source":"ious = bbox_iou(post_train_valid_rois, bbox)\nprint(ious.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.617492Z","start_time":"2019-08-26T03:00:18.966239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Assign labels for each proposal\n\nsame as 2.3.5","metadata":{"ExecuteTime":{"end_time":"2019-08-26T00:49:07.040885Z","start_time":"2019-08-26T00:49:07.036892Z"}}},{"cell_type":"code","source":"bbox_assignments = ious.argmax(axis=1)\nroi_max_ious = ious.max(axis=1)\nroi_target_labels = labels[bbox_assignments]\nprint(roi_target_labels.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.625489Z","start_time":"2019-08-26T03:00:19.619477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Sample pos and neg samples\n\nsame as 2.3.6\n\n- Select the foreground rois as per the pos_iou_thesh. We also want only n_sample x pos_ratio (128 x 0.25 = 32) foreground samples. So incase if we get less than 32 positive samples we will leave it as it is, Incase if we get more than 32 foreground samples, we will sample 32 samples from the positive samples. This is done using the following code.","metadata":{}},{"cell_type":"code","source":"total_n_pos = len(np.where(roi_max_ious >= pos_iou_thresh)[0])\nn_pos_sample = n_sample*pos_ratio if total_n_pos > n_sample*pos_ratio else total_n_pos\nn_neg_sample = n_sample - n_pos_sample\n\nprint(n_pos_sample)\nprint(n_neg_sample)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.635435Z","start_time":"2019-08-26T03:00:19.627483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pos_index = np.where(roi_max_ious >= pos_iou_thresh)[0]\npos_index = np.random.choice(pos_index, size=n_pos_sample, replace=False)\n\nneg_index = np.where((roi_max_ious < neg_iou_thresh_hi) & (roi_max_ious > neg_iou_thresh_lo))[0]\nneg_index = np.random.choice(neg_index, size=n_neg_sample, replace=False)\n\nprint(pos_index.shape)\nprint(neg_index.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.650399Z","start_time":"2019-08-26T03:00:19.63743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Gather positve samples index and negitive samples index, their respective labels and region proposals","metadata":{}},{"cell_type":"code","source":"keep_index = np.append(pos_index, neg_index)\npost_sample_target_labels = roi_target_labels[keep_index].data.numpy()\npost_sample_target_labels[len(pos_index):] = 0\npost_sample_rois = post_train_valid_rois[keep_index]","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.6584Z","start_time":"2019-08-26T03:00:19.653388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pick the ground truth objects for these sample_roi and later parameterize\n\nsame as 2.3.7","metadata":{}},{"cell_type":"code","source":"post_sample_bbox = bbox[bbox_assignments[keep_index]]\npost_sample_format_rois = format_loc(anchors=post_sample_rois, base_anchors=post_sample_bbox.data.numpy())\nprint(post_sample_format_rois.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.669357Z","start_time":"2019-08-26T03:00:19.663362Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fast R-CNN\n\nFast R-CNN used ROI pooling to extract features for each and every proposal suggested by selective search (Fast RCNN) or Region Proposal network (RPN in Faster R- CNN). ","metadata":{}},{"cell_type":"markdown","source":"Region of interest pooling (also known as RoI pooling) purpose is to perform max pooling on inputs of non-uniform sizes to obtain fixed-size feature maps (e.g. 7×7). This layer takes two inputs\n\n- A fixed-size feature map obtained from a deep convolutional network with several convolutions and max-pooling layers\n- An Nx5 matrix of representing a list of regions of interest, where N is the number of RoIs. The first column represents the image index and the remaining four are the co-ordinates of the top left and bottom right corners of the region.","metadata":{}},{"cell_type":"markdown","source":"What does the RoI pooling actually do? For every region of interest from the input list, it takes a section of the input feature map that corresponds to it and scales it to some pre-defined size (e.g., 7×7). The scaling is done by:\n- Dividing the region proposal into equal-sized sections (the number of which is the same as the dimension of the output)\n- Finding the largest value in each section\n- Copying these max values to the output buffer","metadata":{}},{"cell_type":"code","source":"rois = torch.from_numpy(post_sample_rois).float()\nprint(rois.shape)\n# roi_indices = torch.zeros((len(rois),1), dtype=torch.float32)\n# print(rois.shape, roi_indices.shape)\n\n# indices_and_rois = torch.cat([roi_indices, rois], dim=1)\n# print(indices_and_rois.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.679319Z","start_time":"2019-08-26T03:00:19.671339Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now we need to pass this array to the roi_pooling layer. We will briefly discuss the workings of it here. The sudo code is as follows\n\n- Multiply the dimensions of rois with the sub_sampling ratio (16 in this case)\n- Empty output Tensor\n- Take each roi\n    - subset the **feature map** based on the roi dimension\n    - Apply AdaptiveMaxPool2d to this subset Tensor.\n    - Add the outputs to the output Tensor\n- Empty output Tensor goes to the network\n\nWe will define the size to be 7 x 7 and define adaptive_max_pool","metadata":{}},{"cell_type":"markdown","source":"## ROI Pooling layer","metadata":{}},{"cell_type":"code","source":"size = (7, 7)\nadaptive_max_pool = nn.AdaptiveMaxPool2d(size)\n\n# correspond to feature map\nrois.mul_(1/16.0)\nrois = rois.long()","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:19.802257Z","start_time":"2019-08-26T03:00:19.682315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output = []\nnum_rois = len(rois)\nfor roi in rois:\n    roi_feature = output_map[..., roi[0]:roi[2]+1, roi[1]:roi[3]+1]\n    output.append(adaptive_max_pool(roi_feature))\noutput = torch.cat(output, 0)\nprint(output.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:20.121572Z","start_time":"2019-08-26T03:00:19.804253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_ROI_pooling = output.view(output.size(0), -1)\nprint(output_ROI_pooling.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:20.129549Z","start_time":"2019-08-26T03:00:20.123566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loc and Cls layer","metadata":{}},{"cell_type":"code","source":"roi_head = nn.Sequential(nn.Linear(25088, 4096),\n                        nn.Linear(4096, 4096))\n\ncls_loc = nn.Linear(4096, 21*4)\ncls_loc.weight.data.normal_(0, 0.01)\ncls_loc.bias.data.zero_()\n\ncls_score = nn.Linear(4096, 21)\ncls_score.weight.data.normal_(0, 0.01)\ncls_score.bias.data.zero_()","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:21.172784Z","start_time":"2019-08-26T03:00:20.131544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = roi_head(output_ROI_pooling)\nroi_cls_loc = cls_loc(x)\nroi_cls_score = cls_score(x)\n\nprint(roi_cls_loc.shape, roi_cls_score.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:21.59794Z","start_time":"2019-08-26T03:00:21.176757Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Fast R-CNN loss","metadata":{}},{"cell_type":"markdown","source":"- predicted","metadata":{}},{"cell_type":"code","source":"print(roi_cls_loc.shape)\nprint(roi_cls_score.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:21.605933Z","start_time":"2019-08-26T03:00:21.599935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- target","metadata":{}},{"cell_type":"code","source":"print(post_sample_format_rois.shape)\nprint(post_sample_target_labels.shape)\n\ngt_roi_cls_loc = torch.from_numpy(post_sample_format_rois).float()\ngt_roi_cls_label = torch.from_numpy(post_sample_target_labels).long()","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:00:21.615898Z","start_time":"2019-08-26T03:00:21.60891Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### cls loss","metadata":{}},{"cell_type":"code","source":"roi_cls_loss = F.cross_entropy(roi_cls_score, gt_roi_cls_label)\nprint(roi_cls_loss)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:01:07.294447Z","start_time":"2019-08-26T03:01:07.286492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### reg loss","metadata":{}},{"cell_type":"code","source":"num_roi = roi_cls_loc.size(0)\nroi_cls_loc = roi_cls_loc.view(-1, 21, 4)\nroi_cls_loc = roi_cls_loc[torch.arange(num_roi), gt_roi_cls_label]\nprint(roi_cls_loc.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:01:08.21165Z","start_time":"2019-08-26T03:01:08.058534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mask = gt_roi_cls_label>0\nmask_loc_pred = roi_cls_loc[mask]\nmask_loc_target = gt_roi_cls_loc[mask]\n\nprint(mask_loc_pred.shape)\nprint(mask_loc_target.shape)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:01:08.325907Z","start_time":"2019-08-26T03:01:08.315926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = torch.abs(mask_loc_pred-mask_loc_target)\nroi_loc_loss = ((x<0.5).float()*x**2*0.5 + (x>0.5).float()*(x-0.5)).sum()\nprint(roi_loc_loss)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:01:08.519546Z","start_time":"2019-08-26T03:01:08.51158Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### total loss","metadata":{}},{"cell_type":"code","source":"roi_lambda = 10\nN_reg = (gt_roi_cls_label>0).float().sum()\nroi_loss = roi_cls_loss + roi_lambda / N_reg * roi_loc_loss\nprint(roi_loss)","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:01:08.880641Z","start_time":"2019-08-26T03:01:08.87566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Total loss","metadata":{}},{"cell_type":"code","source":"total_loss = rpn_loss + roi_loss","metadata":{"ExecuteTime":{"end_time":"2019-08-26T03:01:10.115228Z","start_time":"2019-08-26T03:01:10.110244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}