{"cells":[{"metadata":{},"cell_type":"markdown","source":"#------------------code for training------------------","execution_count":null},{"metadata":{"_uuid":"ab3c5ade-2631-4273-b148-7bbee7f5c82a","_cell_guid":"1482a63f-6f42-48d5-8ba9-d0c448a4b098","trusted":true},"cell_type":"code","source":"#!pip install easydict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!cp -r ../input/cascadercnn .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cd cascadercnn/lib","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!python setup.py build develop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cd ..","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!ls .","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#lr = 0.00125 for one card and one image per batch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#!python train_cascade_fpn.py --dataset pascal_voc --net res50 --epoch 30 --lr_decay_step 9 --disp_interval 1 --bs 6 --nw 16 --lr 0.001 --lr_decay_step 8 --cuda --mGPUs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!rm -rf ../cascadercnn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#------------------code for testing------------------","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm cascade_fpn_1_64_1686.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport math\nimport os\nimport numpy as np\nimport pandas as pd\n\nos.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\n\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Function\nfrom torch.autograd.function import once_differentiable\nfrom torch.nn.modules.utils import _pair\nfrom torch.nn.functional import avg_pool2d\nfrom torch.autograd import Variable\n\n\nclasses = np.asarray(['__background__', 'wheat'])\npascal_classes = ['__background__', 'wheat']\ncategoryList={'bg':0,'wheat':1} \n\ncfg = { 'ANCHOR_RATIOS': [0.5,1,2], \n 'ANCHOR_SCALES': [4,8,16,32],\n 'FEAT_STRIDE': [16, ],\n 'POOLING_SIZE': 7,\n 'TRAIN_TRUNCATED': False,\n 'POOLING_MODE': 'align',\n 'CROP_RESIZE_WITH_MAX_POOL': False,\n 'FPN_ANCHOR_SCALES': [32, 64, 128, 256, 512],\n 'FPN_FEAT_STRIDES': [4, 8, 16, 32, 64],\n 'FPN_ANCHOR_STRIDE': 1,\n 'RPN_PRE_NMS_TOP_N': 6000,\n 'RPN_POST_NMS_TOP_N': 300,\n 'RPN_NMS_THRESH': 0.7,\n 'RPN_MIN_SIZE': 16,\n 'TRAIN_RPN_NEGATIVE_OVERLAP': 0.3,\n 'TRAIN_RPN_POSITIVE_OVERLAP': 0.7,\n 'TRAIN_RPN_FG_FRACTION': 0.5,\n 'TRAIN_RPN_BATCHSIZE': 256,\n 'TRAIN_RPN_BBOX_INSIDE_WEIGHTS': (1.0, 1.0, 1.0, 1.0),\n 'TRAIN_RPN_POSITIVE_WEIGHT': -1.0,\n 'TRAIN_FG_THRESH': 0.5,\n 'TRAIN_BG_THRESH_HI': 0.5,\n 'TRAIN_BG_THRESH_LO':0.1,\n 'TRAIN_FG_THRESH_2ND': 0.6,\n 'TRAIN_FG_THRESH_3RD': 0.7,\n 'TRAIN_BBOX_NORMALIZE_TARGETS_PRECOMPUTED': True,\n 'TRAIN_BATCH_SIZE': 128,\n 'TRAIN_FG_FRACTION': 0.25,\n 'TRAIN_BBOX_NORMALIZE_MEANS': (0.0, 0.0, 0.0, 0.0),\n 'TRAIN_BBOX_NORMALIZE_STDS':(0.1, 0.1, 0.2, 0.2),\n 'TRAIN_BBOX_INSIDE_WEIGHTS':(1.0, 1.0, 1.0, 1.0),\n 'RESNET_FIXED_BLOCKS': 1,\n #'PIXEL_MEANS': np.array([[[0.485, 0.456, 0.406]]]),\n 'PIXEL_MEANS': np.array([[[122.7717, 115.9465, 102.9801 ]]]), # RGB\n #'PIXEL_MEANS': np.array([[[102.9801, 115.9465, 122.7717]]]), # BGR\n 'TEST_SCALES': (1024,),\n 'TEST_MAX_SIZE':1024,\n 'TEST_BBOX_REG': True,\n \n }\n\n\n#--------------------------------------------------#\ndef clip_boxes(boxes, im_shape, batch_size):\n\n for i in range(batch_size):\n boxes[i,:,0::4].clamp_(0, im_shape[i, 1]-1)\n boxes[i,:,1::4].clamp_(0, im_shape[i, 0]-1)\n boxes[i,:,2::4].clamp_(0, im_shape[i, 1]-1)\n boxes[i,:,3::4].clamp_(0, im_shape[i, 0]-1)\n\n return boxes\n \ndef bbox_transform_inv(boxes, deltas, batch_size):\n # print(\" bbox_transform_inv \")\n # print(\"bbox shape:\",boxes.shape)\n # print(\"deltas shape:\",deltas.shape)\n widths = boxes[:, :, 2] - boxes[:, :, 0] + 1.0\n heights = boxes[:, :, 3] - boxes[:, :, 1] + 1.0\n ctr_x = boxes[:, :, 0] + 0.5 * widths\n ctr_y = boxes[:, :, 1] + 0.5 * heights\n\n dx = deltas[:, :, 0::4]\n dy = deltas[:, :, 1::4]\n dw = deltas[:, :, 2::4]\n dh = deltas[:, :, 3::4]\n\n # print(dx.shape)\n\n pred_ctr_x = dx * widths.unsqueeze(2) + ctr_x.unsqueeze(2)\n pred_ctr_y = dy * heights.unsqueeze(2) + ctr_y.unsqueeze(2)\n pred_w = torch.exp(dw) * widths.unsqueeze(2)\n pred_h = torch.exp(dh) * heights.unsqueeze(2)\n\n pred_boxes = deltas.clone()\n # x1\n pred_boxes[:, :, 0::4] = pred_ctr_x - 0.5 * pred_w\n # y1\n pred_boxes[:, :, 1::4] = pred_ctr_y - 0.5 * pred_h\n # x2\n pred_boxes[:, :, 2::4] = pred_ctr_x + 0.5 * pred_w\n # y2\n pred_boxes[:, :, 3::4] = pred_ctr_y + 0.5 * pred_h\n\n return pred_boxes\n\n\ndef generate_anchors_single_pyramid(scales, ratios, shape, feature_stride, anchor_stride):\n \"\"\"\n scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128]\n ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2]\n shape: [height, width] spatial shape of the feature map over which\n to generate anchors.\n feature_stride: Stride of the feature map relative to the image in pixels.\n anchor_stride: Stride of anchors on the feature map. For example, if the\n value is 2 then generate anchors for every other feature map pixel.\n \"\"\"\n # Get all combinations of scales and ratios\n scales, ratios = np.meshgrid(np.array(scales), np.array(ratios))\n scales = scales.flatten()\n ratios = ratios.flatten()\n\n # Enumerate heights and widths from scales and ratios\n heights = scales / np.sqrt(ratios)\n widths = scales * np.sqrt(ratios)\n\n # Enumerate shifts in feature space\n shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride\n shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride\n shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y)\n\n # Enumerate combinations of shifts, widths, and heights\n box_widths, box_centers_x = np.meshgrid(widths, shifts_x)\n box_heights, box_centers_y = np.meshgrid(heights, shifts_y)\n \n # # Reshape to get a list of (y, x) and a list of (h, w)\n # box_centers = np.stack(\n # [box_centers_y, box_centers_x], axis=2).reshape([-1, 2])\n # box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2])\n\n # NOTE: the original order is (y, x), we changed it to (x, y) for our code\n # Reshape to get a list of (x, y) and a list of (w, h)\n box_centers = np.stack(\n [box_centers_x, box_centers_y], axis=2).reshape([-1, 2])\n box_sizes = np.stack([box_widths, box_heights], axis=2).reshape([-1, 2])\n\n # Convert to corner coordinates (x1, y1, x2, y2)\n boxes = np.concatenate([box_centers - 0.5 * box_sizes,\n box_centers + 0.5 * box_sizes], axis=1)\n # print(boxes)\n return boxes\n\n\ndef generate_anchors_all_pyramids(scales, ratios, feature_shapes, feature_strides,\n anchor_stride):\n \"\"\"Generate anchors at different levels of a feature pyramid. Each scale\n is associated with a level of the pyramid, but each ratio is used in\n all levels of the pyramid.\n Returns:\n anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted\n with the same order of the given scales. So, anchors of scale[0] come\n first, then anchors of scale[1], and so on.\n \"\"\"\n # Anchors\n # [anchor_count, (y1, x1, y2, x2)]\n anchors = []\n for i in range(len(scales)):\n anchors.append(generate_anchors_single_pyramid(scales[i], ratios, feature_shapes[i],\n feature_strides[i], anchor_stride))\n return np.concatenate(anchors, axis=0)\n\n\n\nclass _ProposalTargetLayer(nn.Module):\n \"\"\"\n Assign object detection proposals to ground-truth targets. Produces proposal\n classification labels and bounding-box regression targets.\n \"\"\"\n\n def __init__(self, nclasses):\n super(_ProposalTargetLayer, self).__init__()\n self._num_classes = nclasses\n self.BBOX_NORMALIZE_MEANS = torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS'])\n self.BBOX_NORMALIZE_STDS = torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS'])\n self.BBOX_INSIDE_WEIGHTS = torch.FloatTensor(cfg['TRAIN_BBOX_INSIDE_WEIGHTS'])\n\n def forward(self, all_rois, gt_boxes, num_boxes, stage=1):\n\n self.BBOX_NORMALIZE_MEANS = self.BBOX_NORMALIZE_MEANS.type_as(gt_boxes)\n self.BBOX_NORMALIZE_STDS = self.BBOX_NORMALIZE_STDS.type_as(gt_boxes)\n self.BBOX_INSIDE_WEIGHTS = self.BBOX_INSIDE_WEIGHTS.type_as(gt_boxes)\n\n gt_boxes_append = gt_boxes.new(gt_boxes.size()).zero_()\n gt_boxes_append[:,:,1:5] = gt_boxes[:,:,:4]\n\n # Include ground-truth boxes in the set of candidate rois\n all_rois = torch.cat([all_rois, gt_boxes_append], 1)\n\n num_images = 1\n rois_per_image = int(cfg['TRAIN_BATCH_SIZE'] / num_images)\n fg_rois_per_image = int(np.round(cfg['TRAIN_FG_FRACTION'] * rois_per_image))\n fg_rois_per_image = 1 if fg_rois_per_image == 0 else fg_rois_per_image\n\n labels, rois, gt_assign, bbox_targets, bbox_inside_weights = self._sample_rois_pytorch(\n all_rois, gt_boxes, fg_rois_per_image,\n rois_per_image, self._num_classes, stage=stage)\n\n bbox_outside_weights = (bbox_inside_weights > 0).float()\n\n return rois, labels, gt_assign, bbox_targets, bbox_inside_weights, bbox_outside_weights\n\n def backward(self, top, propagate_down, bottom):\n \"\"\"This layer does not propagate gradients.\"\"\"\n pass\n\n def reshape(self, bottom, top):\n \"\"\"Reshaping happens during the call to forward.\"\"\"\n pass\n\n def _get_bbox_regression_labels_pytorch(self, bbox_target_data, labels_batch, num_classes):\n \"\"\"Bounding-box regression targets (bbox_target_data) are stored in a\n compact form b x N x (class, tx, ty, tw, th)\n\n This function expands those targets into the 4-of-4*K representation used\n by the network (i.e. only one class has non-zero targets).\n\n Returns:\n bbox_target (ndarray): b x N x 4K blob of regression targets\n bbox_inside_weights (ndarray): b x N x 4K blob of loss weights\n \"\"\"\n batch_size = labels_batch.size(0)\n rois_per_image = labels_batch.size(1)\n clss = labels_batch\n bbox_targets = bbox_target_data.new(batch_size, rois_per_image, 4).zero_()\n bbox_inside_weights = bbox_target_data.new(bbox_targets.size()).zero_()\n\n for b in range(batch_size):\n # assert clss[b].sum() > 0\n if clss[b].sum() == 0:\n continue\n inds = torch.nonzero(clss[b] > 0).view(-1)\n for i in range(inds.numel()):\n ind = inds[i]\n bbox_targets[b, ind, :] = bbox_target_data[b, ind, :]\n bbox_inside_weights[b, ind, :] = self.BBOX_INSIDE_WEIGHTS\n\n return bbox_targets, bbox_inside_weights\n\n\n def _compute_targets_pytorch(self, ex_rois, gt_rois):\n \"\"\"Compute bounding-box regression targets for an image.\"\"\"\n\n assert ex_rois.size(1) == gt_rois.size(1)\n assert ex_rois.size(2) == 4\n assert gt_rois.size(2) == 4\n\n batch_size = ex_rois.size(0)\n rois_per_image = ex_rois.size(1)\n\n targets = bbox_transform_batch(ex_rois, gt_rois)\n\n if cfg['TRAIN_BBOX_NORMALIZE_TARGETS_PRECOMPUTED']:\n # Optionally normalize targets by a precomputed mean and stdev\n targets = ((targets - self.BBOX_NORMALIZE_MEANS.expand_as(targets))\n / self.BBOX_NORMALIZE_STDS.expand_as(targets))\n\n return targets\n\n\n def _sample_rois_pytorch(self, all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes, stage=1):\n \"\"\"Generate a random sample of RoIs comprising foreground and background\n examples.\n \"\"\"\n # overlaps: (rois x gt_boxes)\n\n overlaps = bbox_overlaps_batch(all_rois, gt_boxes)\n \n max_overlaps, gt_assignment = torch.max(overlaps, 2)\n\n batch_size = overlaps.size(0)\n num_proposal = overlaps.size(1)\n num_boxes_per_img = overlaps.size(2)\n\n offset = torch.arange(0, batch_size)*gt_boxes.size(1)\n offset = offset.view(-1, 1).type_as(gt_assignment) + gt_assignment\n labels = gt_boxes[:,:,4].contiguous().view(-1)[(offset.view(-1),)].view(batch_size, -1)\n\n labels_batch = labels.new(batch_size, rois_per_image).zero_()\n rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_()\n gt_assign_batch = all_rois.new(batch_size, rois_per_image).zero_()\n gt_rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_()\n # Guard against the case when an image has fewer than max_fg_rois_per_image\n # foreground RoIs\n if stage == 1:\n fg_thresh = cfg['TRAIN_FG_THRESH']\n bg_thresh_hi = cfg['TRAIN_BG_THRESH_HI']\n bg_thresh_lo = cfg['TRAIN_BG_THRESH_LO']\n elif stage == 2:\n fg_thresh = cfg['TRAIN_FG_THRESH_2ND']\n bg_thresh_hi = cfg['TRAIN_FG_THRESH_2ND']\n bg_thresh_lo = cfg['TRAIN_BG_THRESH_LO']\n elif stage == 3:\n fg_thresh = cfg['TRAIN_FG_THRESH_3RD']\n bg_thresh_hi = cfg['TRAIN_FG_THRESH_3RD']\n bg_thresh_lo = cfg['TRAIN_BG_THRESH_LO']\n else:\n raise RuntimeError('stage must be in [1, 2, 3]')\n for i in range(batch_size):\n\n fg_inds = torch.nonzero(max_overlaps[i] >= fg_thresh).view(-1)\n fg_num_rois = fg_inds.numel()\n\n # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)\n bg_inds = torch.nonzero((max_overlaps[i] < bg_thresh_hi) &\n (max_overlaps[i] >= bg_thresh_lo)).view(-1)\n bg_num_rois = bg_inds.numel()\n\n if fg_num_rois > 0 and bg_num_rois > 0:\n # sampling fg\n fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois)\n\n # torch.randperm seems has a bug on multi-gpu setting that cause the segfault.\n # See https://github.com/pytorch/pytorch/issues/1868 for more details.\n # use numpy instead.\n #rand_num = torch.randperm(fg_num_rois).long().cuda()\n rand_num = torch.from_numpy(np.random.permutation(fg_num_rois)).type_as(gt_boxes).long()\n fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]]\n\n # sampling bg\n bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image\n\n # Seems torch.rand has a bug, it will generate very large number and make an error.\n # We use numpy rand instead.\n #rand_num = (torch.rand(bg_rois_per_this_image) * bg_num_rois).long().cuda()\n rand_num = np.floor(np.random.rand(bg_rois_per_this_image) * bg_num_rois)\n rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long()\n bg_inds = bg_inds[rand_num]\n\n elif fg_num_rois > 0 and bg_num_rois == 0:\n # sampling fg\n #rand_num = torch.floor(torch.rand(rois_per_image) * fg_num_rois).long().cuda()\n rand_num = np.floor(np.random.rand(rois_per_image) * fg_num_rois)\n rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long()\n fg_inds = fg_inds[rand_num]\n fg_rois_per_this_image = rois_per_image\n bg_rois_per_this_image = 0\n elif bg_num_rois > 0 and fg_num_rois == 0:\n # sampling bg\n #rand_num = torch.floor(torch.rand(rois_per_image) * bg_num_rois).long().cuda()\n rand_num = np.floor(np.random.rand(rois_per_image) * bg_num_rois)\n rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long()\n\n bg_inds = bg_inds[rand_num]\n bg_rois_per_this_image = rois_per_image\n fg_rois_per_this_image = 0\n else:\n print(i, overlaps[i], max_overlaps[i], gt_boxes[i])\n raise ValueError(\"bg_num_rois = 0 and fg_num_rois = 0, this should not happen!\")\n\n # The indices that we're selecting (both fg and bg)\n keep_inds = torch.cat([fg_inds, bg_inds], 0)\n\n # Select sampled values from various arrays:\n labels_batch[i].copy_(labels[i][keep_inds])\n\n # Clamp labels for the background RoIs to 0\n if fg_rois_per_this_image < rois_per_image:\n labels_batch[i][fg_rois_per_this_image:] = 0\n\n rois_batch[i] = all_rois[i][keep_inds]\n rois_batch[i,:,0] = i\n\n # TODO: check the below line when batch_size > 1, no need to add offset here\n gt_assign_batch[i] = gt_assignment[i][keep_inds]\n\n gt_rois_batch[i] = gt_boxes[i][gt_assignment[i][keep_inds]]\n\n bbox_target_data = self._compute_targets_pytorch(\n rois_batch[:,:,1:5], gt_rois_batch[:,:,:4])\n\n bbox_targets, bbox_inside_weights = \\\n self._get_bbox_regression_labels_pytorch(bbox_target_data, labels_batch, num_classes)\n\n return labels_batch, rois_batch, gt_assign_batch, bbox_targets, bbox_inside_weights\n \nclass _AnchorTargetLayer_FPN(nn.Module):\n \"\"\"\n Assign anchors to ground-truth targets. Produces anchor classification\n labels and bounding-box regression targets.\n \"\"\"\n def __init__(self, feat_stride, scales, ratios):\n super(_AnchorTargetLayer_FPN, self).__init__()\n self._anchor_ratios = ratios\n self._feat_stride = feat_stride\n self._fpn_scales = np.array(cfg['FPN_ANCHOR_SCALES'])\n self._fpn_feature_strides = np.array(cfg['FPN_FEAT_STRIDES'])\n self._fpn_anchor_stride = cfg['FPN_ANCHOR_STRIDE']\n\n # allow boxes to sit over the edge by a small amount\n self._allowed_border = 0 # default is 0\n\n def forward(self, input):\n # Algorithm:\n #\n # for each (H, W) location i\n # generate 9 anchor boxes centered on cell i\n # apply predicted bbox deltas at cell i to each of the 9 anchors\n # filter out-of-image anchors\n # \n scores = input[0]\n gt_boxes = input[1]\n im_info = input[2]\n num_boxes = input[3]\n feat_shapes = input[4]\n\n # NOTE: need to change\n # height, width = scores.size(2), scores.size(3)\n height, width = 0, 0\n\n batch_size = gt_boxes.size(0)\n\n anchors = torch.from_numpy(generate_anchors_all_pyramids(self._fpn_scales, self._anchor_ratios, \n feat_shapes, self._fpn_feature_strides, self._fpn_anchor_stride)).type_as(scores) \n total_anchors = anchors.size(0)\n # print(self._fpn_feature_strides)\n # print(anchors.shape)\n keep = ((anchors[:, 0] >= -self._allowed_border) &\n (anchors[:, 1] >= -self._allowed_border) &\n (anchors[:, 2] < long(im_info[0][1]) + self._allowed_border) &\n (anchors[:, 3] < long(im_info[0][0]) + self._allowed_border))\n\n inds_inside = torch.nonzero(keep).view(-1)\n\n # keep only inside anchors\n anchors = anchors[inds_inside, :]\n\n # label: 1 is positive, 0 is negative, -1 is dont care\n labels = gt_boxes.new(batch_size, inds_inside.size(0)).fill_(-1)\n bbox_inside_weights = gt_boxes.new(batch_size, inds_inside.size(0)).zero_()\n bbox_outside_weights = gt_boxes.new(batch_size, inds_inside.size(0)).zero_()\n\n overlaps = bbox_overlaps_batch(anchors, gt_boxes)\n\n max_overlaps, argmax_overlaps = torch.max(overlaps, 2)\n gt_max_overlaps, _ = torch.max(overlaps, 1)\n\n labels[max_overlaps < cfg['TRAIN_RPN_NEGATIVE_OVERLAP']] = 0\n\n gt_max_overlaps[gt_max_overlaps==0] = 1e-5\n keep = torch.sum(overlaps.eq(gt_max_overlaps.view(batch_size,1,-1).expand_as(overlaps)), 2)\n\n if torch.sum(keep) > 0:\n labels[keep>0] = 1\n\n # fg label: above threshold IOU\n labels[max_overlaps >= cfg['TRAIN_RPN_POSITIVE_OVERLAP']] = 1\n\n num_fg = int(cfg['TRAIN_RPN_FG_FRACTION'] * cfg['TRAIN_RPN_BATCHSIZE'])\n\n sum_fg = torch.sum((labels == 1).int(), 1)\n sum_bg = torch.sum((labels == 0).int(), 1)\n\n for i in range(batch_size):\n # subsample positive labels if we have too many\n if sum_fg[i] > num_fg:\n fg_inds = torch.nonzero(labels[i] == 1).view(-1)\n # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. \n # See https://github.com/pytorch/pytorch/issues/1868 for more details.\n # use numpy instead. \n #rand_num = torch.randperm(fg_inds.size(0)).type_as(gt_boxes).long()\n rand_num = torch.from_numpy(np.random.permutation(fg_inds.size(0))).type_as(gt_boxes).long()\n disable_inds = fg_inds[rand_num[:fg_inds.size(0)-num_fg]]\n labels[i][disable_inds] = -1\n\n num_bg = cfg['TRAIN_RPN_BATCHSIZE'] - sum_fg[i]\n\n # subsample negative labels if we have too many\n if sum_bg[i] > num_bg:\n bg_inds = torch.nonzero(labels[i] == 0).view(-1)\n #rand_num = torch.randperm(bg_inds.size(0)).type_as(gt_boxes).long()\n\n rand_num = torch.from_numpy(np.random.permutation(bg_inds.size(0))).type_as(gt_boxes).long()\n disable_inds = bg_inds[rand_num[:bg_inds.size(0)-num_bg]]\n labels[i][disable_inds] = -1\n\n offset = torch.arange(0, batch_size)*gt_boxes.size(1)\n\n argmax_overlaps = argmax_overlaps + offset.view(batch_size, 1).type_as(argmax_overlaps)\n bbox_targets = _compute_targets_batch(anchors, gt_boxes.view(-1,5)[argmax_overlaps.view(-1), :].view(batch_size, -1, 5))\n\n # use a single value instead of 4 values for easy index.\n bbox_inside_weights[labels==1] = cfg['TRAIN_RPN_BBOX_INSIDE_WEIGHTS'][0]\n\n\n if cfg['TRAIN_RPN_POSITIVE_WEIGHT'] < 0:\n num_examples = torch.sum(labels[i] >= 0)\n positive_weights = 1.0 / num_examples.item()\n negative_weights = 1.0 / num_examples.item()\n else:\n assert ((cfg['TRAIN_RPN_POSITIVE_WEIGHT'] > 0) &\n (cfg['TRAIN_RPN_POSITIVE_WEIGHT'] < 1))\n\n bbox_outside_weights[labels == 1] = positive_weights\n bbox_outside_weights[labels == 0] = negative_weights\n\n labels = _unmap(labels, total_anchors, inds_inside, batch_size, fill=-1)\n bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, batch_size, fill=0)\n bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, batch_size, fill=0)\n bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, batch_size, fill=0)\n\n outputs = []\n\n # labels = labels.view(batch_size, height, width, A).permute(0,3,1,2).contiguous()\n # labels = labels.view(batch_size, 1, A * height, width)\n outputs.append(labels)\n # bbox_targets = bbox_targets.view(batch_size, height, width, A*4).permute(0,3,1,2).contiguous()\n outputs.append(bbox_targets)\n\n # anchors_count = bbox_inside_weights.size(1)\n # bbox_inside_weights = bbox_inside_weights.view(batch_size,anchors_count,1).expand(batch_size, anchors_count, 4)\n # bbox_inside_weights = bbox_inside_weights.contiguous().view(batch_size, height, width, 4*A)\\\n # .permute(0,3,1,2).contiguous()\n\n outputs.append(bbox_inside_weights)\n\n # bbox_outside_weights = bbox_outside_weights.view(batch_size,anchors_count,1).expand(batch_size, anchors_count, 4)\n # bbox_outside_weights = bbox_outside_weights.contiguous().view(batch_size, height, width, 4*A)\\\n # .permute(0,3,1,2).contiguous()\n outputs.append(bbox_outside_weights)\n\n return outputs\n\n def backward(self, top, propagate_down, bottom):\n \"\"\"This layer does not propagate gradients.\"\"\"\n pass\n\n def reshape(self, bottom, top):\n \"\"\"Reshaping happens during the call to forward.\"\"\"\n pass\n\n \n \nclass _ProposalLayer_FPN(nn.Module):\n \"\"\"\n Outputs object detection proposals by applying estimated bounding-box\n transformations to a set of regular boxes (called \"anchors\").\n \"\"\"\n\n def __init__(self, feat_stride, scales, ratios):\n super(_ProposalLayer_FPN, self).__init__()\n self._anchor_ratios = ratios\n self._feat_stride = feat_stride\n self._fpn_scales = np.array(cfg['FPN_ANCHOR_SCALES'])\n self._fpn_feature_strides = np.array(cfg['FPN_FEAT_STRIDES'])\n self._fpn_anchor_stride = cfg['FPN_ANCHOR_STRIDE']\n # self._anchors = torch.from_numpy(generate_anchors_all_pyramids(self._fpn_scales, ratios, self._fpn_feature_strides, fpn_anchor_stride))\n # self._num_anchors = self._anchors.size(0)\n\n def forward(self, input):\n\n # Algorithm:\n #\n # for each (H, W) location i\n # generate A anchor boxes centered on cell i\n # apply predicted bbox deltas at cell i to each of the A anchors\n # clip predicted boxes to image\n # remove predicted boxes with either height or width < threshold\n # sort all (proposal, score) pairs by score from highest to lowest\n # take top pre_nms_topN proposals before NMS\n # apply NMS with threshold 0.7 to remaining proposals\n # take after_nms_topN proposals after NMS\n # return the top proposals (-> RoIs top, scores top)\n\n\n # the first set of _num_anchors channels are bg probs\n # the second set are the fg probs\n scores = input[0][:, :, 1] # batch_size x num_rois x 1\n bbox_deltas = input[1] # batch_size x num_rois x 4\n im_info = input[2]\n cfg_key = input[3]\n feat_shapes = input[4] \n\n pre_nms_topN = cfg['RPN_PRE_NMS_TOP_N']\n post_nms_topN = cfg['RPN_POST_NMS_TOP_N']\n nms_thresh = cfg['RPN_NMS_THRESH']\n min_size = cfg['RPN_MIN_SIZE']\n\n batch_size = bbox_deltas.size(0)\n\n anchors = torch.from_numpy(generate_anchors_all_pyramids(self._fpn_scales, self._anchor_ratios, \n feat_shapes, self._fpn_feature_strides, self._fpn_anchor_stride)).type_as(scores)\n num_anchors = anchors.size(0)\n\n anchors = anchors.view(1, num_anchors, 4).expand(batch_size, num_anchors, 4)\n\n # Convert anchors into proposals via bbox transformations\n proposals = bbox_transform_inv(anchors, bbox_deltas, batch_size)\n\n # 2. clip predicted boxes to image\n proposals = clip_boxes(proposals, im_info, batch_size)\n # keep_idx = self._filter_boxes(proposals, min_size).squeeze().long().nonzero().squeeze()\n \n scores_keep = scores\n proposals_keep = proposals\n\n _, order = torch.sort(scores_keep, 1, True)\n\n output = scores.new(batch_size, post_nms_topN, 5).zero_()\n for i in range(batch_size):\n # # 3. remove predicted boxes with either height or width < threshold\n # # (NOTE: convert min_size to input image scale stored in im_info[2])\n proposals_single = proposals_keep[i]\n scores_single = scores_keep[i]\n\n # # 4. sort all (proposal, score) pairs by score from highest to lowest\n # # 5. take top pre_nms_topN (e.g. 6000)\n order_single = order[i]\n\n if pre_nms_topN > 0 and pre_nms_topN < scores_keep.numel():\n order_single = order_single[:pre_nms_topN]\n\n proposals_single = proposals_single[order_single, :]\n scores_single = scores_single[order_single].view(-1,1)\n # print(\"-------------------------\")\n # print(type(proposals_single))\n # print(proposals_single.shape)\n # print(type(scores_single))\n # print(scores_single.shape)\n # print(\"-------------------------\")\n # # 6. apply nms (e.g. threshold = 0.7)\n # # 7. take after_nms_topN (e.g. 300)\n # # 8. return the top proposals (-> RoIs top)\n # print(proposals_single)\n # print('------------------------')\n # print(proposals_single.cpu().numpy())\n keep_idx_i = soft_nms(proposals_single, scores_single.squeeze(1), sigma=0.5, thresh=0.001, cuda=1)\n # keep_idx_i = soft_nms(proposals_single.cpu().numpy(), scores_single.cpu().numpy(), 0, thresh = 0.2, Nt = nms_thresh)\n # keep_idx_i = torch.from_numpy(keep_idx_i)\n # keep_idx_i = nms(proposals_single, scores_single.squeeze(1), nms_thresh)\n # keep_idx_i = nms(proposals_single, scores_single, nms_thresh)\n keep_idx_i = keep_idx_i.long().view(-1)\n\n if post_nms_topN > 0:\n keep_idx_i = keep_idx_i[:post_nms_topN]\n proposals_single = proposals_single[keep_idx_i, :]\n scores_single = scores_single[keep_idx_i, :]\n\n # padding 0 at the end.\n num_proposal = proposals_single.size(0)\n output[i,:,0] = i\n output[i,:num_proposal,1:] = proposals_single\n\n return output\n\n def backward(self, top, propagate_down, bottom):\n \"\"\"This layer does not propagate gradients.\"\"\"\n pass\n\n def reshape(self, bottom, top):\n \"\"\"Reshaping happens during the call to forward.\"\"\"\n pass\n\n def _filter_boxes(self, boxes, min_size):\n \"\"\"Remove all boxes with any side smaller than min_size.\"\"\"\n ws = boxes[:, :, 2] - boxes[:, :, 0] + 1\n hs = boxes[:, :, 3] - boxes[:, :, 1] + 1\n keep = ((ws >= min_size) & (hs >= min_size))\n return keep\n\n\nclass _RPN_FPN(nn.Module):\n \"\"\" region proposal network \"\"\"\n def __init__(self, din):\n super(_RPN_FPN, self).__init__()\n\n self.din = din # get depth of input feature map, e.g., 512\n self.anchor_ratios = cfg['ANCHOR_RATIOS']\n self.anchor_scales = cfg['ANCHOR_SCALES']\n self.feat_stride = cfg['FEAT_STRIDE']\n\n # define the convrelu layers processing input feature map\n self.RPN_Conv = nn.Conv2d(self.din, 512, 3, 1, 1, bias=True)\n\n # define bg/fg classifcation score layer\n # self.nc_score_out = len(self.anchor_scales) * len(self.anchor_ratios) * 2 # 2(bg/fg) * 9 (anchors)\n self.nc_score_out = 1 * len(self.anchor_ratios) * 2 # 2(bg/fg) * 3 (anchor ratios) * 1 (anchor scale)\n self.RPN_cls_score = nn.Conv2d(512, self.nc_score_out, 1, 1, 0)\n\n # define anchor box offset prediction layer\n # self.nc_bbox_out = len(self.anchor_scales) * len(self.anchor_ratios) * 4 # 4(coords) * 9 (anchors)\n self.nc_bbox_out = 1 * len(self.anchor_ratios) * 4 # 4(coords) * 3 (anchors) * 1 (anchor scale)\n self.RPN_bbox_pred = nn.Conv2d(512, self.nc_bbox_out, 1, 1, 0)\n\n # define proposal layer\n self.RPN_proposal = _ProposalLayer_FPN(self.feat_stride, self.anchor_scales, self.anchor_ratios)\n\n # define anchor target layer\n self.RPN_anchor_target = _AnchorTargetLayer_FPN(self.feat_stride, self.anchor_scales, self.anchor_ratios)\n\n self.rpn_loss_cls = 0\n self.rpn_loss_box = 0\n\n @staticmethod\n def reshape(x, d):\n input_shape = x.size()\n x = x.contiguous().view(\n input_shape[0],\n int(d),\n int(float(input_shape[1] * input_shape[2]) / float(d)),\n input_shape[3]\n )\n return x\n\n def forward(self, rpn_feature_maps, im_info, gt_boxes, num_boxes): \n\n n_feat_maps = len(rpn_feature_maps)\n\n rpn_cls_scores = []\n rpn_cls_probs = []\n rpn_bbox_preds = []\n rpn_shapes = []\n\n for i in range(n_feat_maps):\n feat_map = rpn_feature_maps[i]\n batch_size = feat_map.size(0)\n \n # return feature map after convrelu layer\n rpn_conv1 = F.relu(self.RPN_Conv(feat_map), inplace=True)\n # get rpn classification score\n rpn_cls_score = self.RPN_cls_score(rpn_conv1)\n\n rpn_cls_score_reshape = self.reshape(rpn_cls_score, 2)\n rpn_cls_prob_reshape = F.softmax(rpn_cls_score_reshape, 1)\n rpn_cls_prob = self.reshape(rpn_cls_prob_reshape, self.nc_score_out)\n\n # get rpn offsets to the anchor boxes\n rpn_bbox_pred = self.RPN_bbox_pred(rpn_conv1)\n\n rpn_shapes.append([rpn_cls_score.size()[2], rpn_cls_score.size()[3]])\n rpn_cls_scores.append(rpn_cls_score.permute(0, 2, 3, 1).contiguous().view(batch_size, -1, 2))\n rpn_cls_probs.append(rpn_cls_prob.permute(0, 2, 3, 1).contiguous().view(batch_size, -1, 2))\n rpn_bbox_preds.append(rpn_bbox_pred.permute(0, 2, 3, 1).contiguous().view(batch_size, -1, 4))\n\n rpn_cls_score_alls = torch.cat(rpn_cls_scores, 1)\n rpn_cls_prob_alls = torch.cat(rpn_cls_probs, 1)\n rpn_bbox_pred_alls = torch.cat(rpn_bbox_preds, 1)\n\n n_rpn_pred = rpn_cls_score_alls.size(1)\n\n # proposal layer\n cfg_key = 'TRAIN' if self.training else 'TEST'\n\n rois = self.RPN_proposal((rpn_cls_prob_alls.data, rpn_bbox_pred_alls.data,\n im_info, cfg_key, rpn_shapes))\n\n self.rpn_loss_cls = 0\n self.rpn_loss_box = 0\n\n # generating training labels and build the rpn loss\n if self.training:\n assert gt_boxes is not None\n\n rpn_data = self.RPN_anchor_target((rpn_cls_score_alls.data, gt_boxes, im_info, num_boxes, rpn_shapes))\n\n # compute classification loss\n rpn_label = rpn_data[0].view(batch_size, -1)\n rpn_keep = Variable(rpn_label.view(-1).ne(-1).nonzero().view(-1))\n rpn_cls_score = torch.index_select(rpn_cls_score_alls.view(-1,2), 0, rpn_keep)\n rpn_label = torch.index_select(rpn_label.view(-1), 0, rpn_keep.data)\n rpn_label = Variable(rpn_label.long())\n self.rpn_loss_cls = F.cross_entropy(rpn_cls_score, rpn_label)\n fg_cnt = torch.sum(rpn_label.data.ne(0))\n\n rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = rpn_data[1:]\n # print(rpn_bbox_targets.shape)\n # compute bbox regression loss\n rpn_bbox_inside_weights = Variable(rpn_bbox_inside_weights.unsqueeze(2) \\\n .expand(batch_size, rpn_bbox_inside_weights.size(1), 4))\n rpn_bbox_outside_weights = Variable(rpn_bbox_outside_weights.unsqueeze(2) \\\n .expand(batch_size, rpn_bbox_outside_weights.size(1), 4))\n rpn_bbox_targets = Variable(rpn_bbox_targets)\n \n self.rpn_loss_box = _smooth_l1_loss(rpn_bbox_pred_alls, rpn_bbox_targets, rpn_bbox_inside_weights, \n rpn_bbox_outside_weights, sigma=3)\n\n return rois, self.rpn_loss_cls, self.rpn_loss_box\n\n\nclass _ROIPool(Function):\n @staticmethod\n def forward(ctx, input, roi, output_size, spatial_scale):\n ctx.output_size = _pair(output_size)\n ctx.spatial_scale = spatial_scale\n ctx.input_shape = input.size()\n output, argmax = _C.roi_pool_forward(\n input, roi, spatial_scale, output_size[0], output_size[1]\n )\n ctx.save_for_backward(input, roi, argmax)\n return output\n\n @staticmethod\n @once_differentiable\n def backward(ctx, grad_output):\n input, rois, argmax = ctx.saved_tensors\n output_size = ctx.output_size\n spatial_scale = ctx.spatial_scale\n bs, ch, h, w = ctx.input_shape\n grad_input = _C.roi_pool_backward(\n grad_output,\n input,\n rois,\n argmax,\n spatial_scale,\n output_size[0],\n output_size[1],\n bs,\n ch,\n h,\n w,\n )\n return grad_input, None, None, None\n\n\nroi_pool = _ROIPool.apply\n\n\nclass ROIPool(nn.Module):\n def __init__(self, output_size, spatial_scale):\n super(ROIPool, self).__init__()\n self.output_size = output_size\n self.spatial_scale = spatial_scale\n\n def forward(self, input, rois):\n return roi_pool(input, rois, self.output_size, self.spatial_scale)\n\n def __repr__(self):\n tmpstr = self.__class__.__name__ + \"(\"\n tmpstr += \"output_size=\" + str(self.output_size)\n tmpstr += \", spatial_scale=\" + str(self.spatial_scale)\n tmpstr += \")\"\n return tmpstr\n\n\nclass _ROIAlign(Function):\n @staticmethod\n def forward(ctx, input, roi, output_size, spatial_scale, sampling_ratio):\n ctx.save_for_backward(roi)\n ctx.output_size = _pair(output_size)\n ctx.spatial_scale = spatial_scale\n ctx.sampling_ratio = sampling_ratio\n ctx.input_shape = input.size()\n output = _C.roi_align_forward(input, roi, spatial_scale, output_size[0], output_size[1], sampling_ratio)\n return output\n\n @staticmethod\n @once_differentiable\n def backward(ctx, grad_output):\n rois, = ctx.saved_tensors\n output_size = ctx.output_size\n spatial_scale = ctx.spatial_scale\n sampling_ratio = ctx.sampling_ratio\n bs, ch, h, w = ctx.input_shape\n grad_input = _C.roi_align_backward(\n grad_output,\n rois,\n spatial_scale,\n output_size[0],\n output_size[1],\n bs,\n ch,\n h,\n w,\n sampling_ratio,\n )\n return grad_input, None, None, None, None\n\nroi_align = _ROIAlign.apply\n\n\ndef bbox_decode(rois, bbox_pred, batch_size, class_agnostic, classes, im_info, training, cls_prob):\n boxes = rois.data[:, :, 1:5]\n if cfg['TEST_BBOX_REG']:\n # Apply bounding-box regression deltas\n box_deltas = bbox_pred.data\n if cfg['TRAIN_BBOX_NORMALIZE_TARGETS_PRECOMPUTED']:\n # Optionally normalize targets by a precomputed mean and stdev\n if class_agnostic or training:\n box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS']).cuda() \\\n + torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS']).cuda()\n box_deltas = box_deltas.view(batch_size, -1, 4)\n else:\n cls_prob[:,0]=0\n bbox_pred_cls_argmax=torch.argmax(cls_prob,dim=1)\n # print(bbox_pred_cls_argmax)\n for i in range(bbox_pred.size(1)):\n bbox_pred_cls_argmax[i]=bbox_pred_cls_argmax[i]+i*classes\n\n bbox_pred_max=bbox_pred.view(batch_size,-1,4)\n bbox_pred_max=torch.index_select(bbox_pred_max,1,bbox_pred_cls_argmax)\n box_deltas = bbox_pred_max.data\n\n box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS']).cuda() \\\n + torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS']).cuda()\n box_deltas = box_deltas.view(batch_size, -1,4)\n\n pred_boxes = bbox_transform_inv(boxes, box_deltas, batch_size)\n pred_boxes = clip_boxes(pred_boxes, im_info, batch_size)\n else:\n # Simply repeat the boxes, once for each class\n pred_boxes = boxes\n\n pred_boxes = pred_boxes.view(batch_size,-1,4)\n ret_boxes = torch.zeros(pred_boxes.size(0), pred_boxes.size(1), 5).cuda()\n ret_boxes[:, :, 1:] = pred_boxes\n for b in range(batch_size):\n ret_boxes[b, :, 0] = b\n\n return ret_boxes\n\n\nclass ROIAlignAvg(nn.Module):\n def __init__(self, output_size, spatial_scale, sampling_ratio):\n super(ROIAlignAvg, self).__init__()\n self.output_size = output_size\n self.spatial_scale = spatial_scale\n self.sampling_ratio = sampling_ratio\n\n def forward(self, input, rois,spatial_scale):\n self.spatial_scale = spatial_scale\n # x= roi_align(\n # input, rois, self.output_size, self.spatial_scale, self.sampling_ratio\n # )\n x= torchvision.ops.roi_align(input, rois, self.output_size, self.spatial_scale, self.sampling_ratio)\n return avg_pool2d(x, kernel_size=2, stride=1)\n\n def __repr__(self):\n tmpstr = self.__class__.__name__ + \"(\"\n tmpstr += \"output_size=\" + str(self.output_size)\n tmpstr += \", spatial_scale=\" + str(self.spatial_scale)\n tmpstr += \", sampling_ratio=\" + str(self.sampling_ratio)\n tmpstr += \")\"\n return tmpstr \n\n \nclass _FPN(nn.Module):\n \"\"\" FPN \"\"\"\n def __init__(self, classes, class_agnostic):\n super(_FPN, self).__init__()\n self.classes = classes\n self.n_classes = len(classes)\n self.class_agnostic = class_agnostic\n # loss\n self.RCNN_loss_cls = 0\n self.RCNN_loss_bbox = 0\n\n self.maxpool2d = nn.MaxPool2d(1, stride=2)\n # define rpn\n self.RCNN_rpn = _RPN_FPN(self.dout_base_model)\n self.RCNN_proposal_target = _ProposalTargetLayer(self.n_classes)\n\n # NOTE: the original paper used pool_size = 7 for cls branch, and 14 for mask branch, to save the\n # computation time, we first use 14 as the pool_size, and then do stride=2 pooling for cls branch.\n self.RCNN_roi_pool = ROIPool((cfg['POOLING_SIZE'], cfg['POOLING_SIZE']), 1.0/16.0)\n self.RCNN_roi_align = ROIAlignAvg((cfg['POOLING_SIZE']+1, cfg['POOLING_SIZE']+1), 1.0/16.0, 0)\n # self.RCNN_roi_pool = ROIPool(cfg['POOLING_SIZE'], cfg['POOLING_SIZE'], 1.0/16.0)\n # self.RCNN_roi_align = ROIAlignAvg(cfg['POOLING_SIZE'], cfg['POOLING_SIZE'], 1.0/16.0)\n self.grid_size = cfg['POOLING_SIZE'] * 2 if cfg['CROP_RESIZE_WITH_MAX_POOL'] else cfg['POOLING_SIZE']\n\n def _init_weights(self):\n def normal_init(m, mean, stddev, truncated=False):\n \"\"\"\n weight initalizer: truncated normal and random normal.\n \"\"\"\n # x is a parameter\n if truncated:\n m.weight.data.normal_().fmod_(2).mul_(stddev).add_(mean) # not a perfect approximation\n else:\n m.weight.data.normal_(mean, stddev)\n m.bias.data.zero_()\n\n # custom weights initialization called on netG and netD\n def weights_init(m, mean, stddev, truncated=False):\n classname = m.__class__.__name__\n if classname.find('Conv') != -1:\n m.weight.data.normal_(0.0, 0.02)\n m.bias.data.fill_(0)\n elif classname.find('BatchNorm') != -1:\n m.weight.data.normal_(1.0, 0.02)\n m.bias.data.fill_(0)\n\n normal_init(self.RCNN_toplayer, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_smooth1, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_smooth2, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_smooth3, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_latlayer1, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_latlayer2, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_latlayer3, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n\n normal_init(self.RCNN_rpn.RPN_Conv, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_rpn.RPN_cls_score, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_rpn.RPN_bbox_pred, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_cls_score, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_bbox_pred, 0, 0.001, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_cls_score_2nd, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_bbox_pred_2nd, 0, 0.001, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_cls_score_3rd, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n normal_init(self.RCNN_bbox_pred_3rd, 0, 0.001, cfg['TRAIN_TRUNCATED'])\n weights_init(self.RCNN_top, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n weights_init(self.RCNN_top_2nd, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n weights_init(self.RCNN_top_3rd, 0, 0.01, cfg['TRAIN_TRUNCATED'])\n\n def create_architecture(self):\n self._init_modules()\n self._init_weights()\n\n def _upsample_add(self, x, y):\n '''Upsample and add two feature maps.\n Args:\n x: (Variable) top feature map to be upsampled.\n y: (Variable) lateral feature map.\n Returns:\n (Variable) added feature map.\n Note in PyTorch, when input size is odd, the upsampled feature map\n with `F.upsample(..., scale_factor=2, mode='nearest')`\n maybe not equal to the lateral feature map size.\n e.g.\n original input size: [N,_,15,15] ->\n conv2d feature map size: [N,_,8,8] ->\n upsampled feature map size: [N,_,16,16]\n So we choose bilinear upsample which supports arbitrary output sizes.\n '''\n _,_,H,W = y.size()\n return F.interpolate(x, size=(H,W), mode='bilinear',align_corners=True) + y\n\n def _PyramidRoI_Feat(self, feat_maps, rois, im_info):\n ''' roi pool on pyramid feature maps'''\n # do roi pooling based on predicted rois\n # print(\"rois shape\",rois.shape)\n # print(\"feat_maps\",feat_maps.shape)\n img_area = im_info[0][0] * im_info[0][1]\n h = rois.data[:, 4] - rois.data[:, 2] + 1\n w = rois.data[:, 3] - rois.data[:, 1] + 1\n # print(h)\n # print(w)\n \n roi_level = torch.log2(torch.sqrt(h * w) / 224.0)\n roi_level = torch.round(roi_level + 4)\n roi_level[roi_level < 2] = 2\n roi_level[roi_level > 5] = 5\n # roi_level.fill_(5)\n # print(\"roi_level\",roi_level)\n if cfg['POOLING_MODE'] == 'align':\n roi_pool_feats = []\n box_to_levels = []\n for i, l in enumerate(range(2, 6)):\n # print(i, l)\n # print(roi_level)\n if (roi_level == l).sum() == 0:\n continue\n \n idx_l = (roi_level == l).nonzero().squeeze()\n # print(idx_l.dim())\n # print((idx_l.cpu().numpy()))\n if(idx_l.dim()==0):\n idx_l=idx_l.unsqueeze(0)\n # continue\n # print(\"^^^^^^^^^^^^^^^^^^^^^^\",idx_l.dim())\n box_to_levels.append(idx_l)\n scale = feat_maps[i].size(2) / im_info[0][0]\n # self.RCNN_roi_align.scale=scale\n feat = self.RCNN_roi_align(feat_maps[i], rois[idx_l], scale)\n roi_pool_feats.append(feat)\n\n # print(\"box_to_levels\")\n # print(box_to_levels)\n roi_pool_feat = torch.cat(roi_pool_feats, 0)\n box_to_level = torch.cat(box_to_levels, 0)\n idx_sorted, order = torch.sort(box_to_level)\n roi_pool_feat = roi_pool_feat[order]\n\n elif cfg['POOLING_MODE'] == 'pool':\n roi_pool_feats = []\n box_to_levels = []\n for i, l in enumerate(range(2, 6)):\n if (roi_level == l).sum() == 0:\n continue\n idx_l = (roi_level == l).nonzero().squeeze()\n box_to_levels.append(idx_l)\n scale = feat_maps[i].size(2) / im_info[0][0]\n self.RCNN_roi_pool.scale=scale\n feat = self.RCNN_roi_pool(feat_maps[i], rois[idx_l])\n roi_pool_feats.append(feat)\n roi_pool_feat = torch.cat(roi_pool_feats, 0)\n box_to_level = torch.cat(box_to_levels, 0)\n idx_sorted, order = torch.sort(box_to_level)\n roi_pool_feat = roi_pool_feat[order]\n \n return roi_pool_feat\n\n def forward(self, im_data, im_info, gt_boxes, num_boxes):\n batch_size = im_data.size(0)\n\n im_info = im_info.data\n gt_boxes = gt_boxes.data\n num_boxes = num_boxes.data\n\n # feed image data to base model to obtain base feature map\n # Bottom-up\n c1 = self.RCNN_layer0(im_data)\n c2 = self.RCNN_layer1(c1)\n c3 = self.RCNN_layer2(c2)\n c4 = self.RCNN_layer3(c3)\n c5 = self.RCNN_layer4(c4)\n # Top-down\n p5 = self.RCNN_toplayer(c5)\n p4 = self._upsample_add(p5, self.RCNN_latlayer1(c4))\n p4 = self.RCNN_smooth1(p4)\n p3 = self._upsample_add(p4, self.RCNN_latlayer2(c3))\n p3 = self.RCNN_smooth2(p3)\n p2 = self._upsample_add(p3, self.RCNN_latlayer3(c2))\n p2 = self.RCNN_smooth3(p2)\n\n p6 = self.maxpool2d(p5)\n\n rpn_feature_maps = [p2, p3, p4, p5, p6]\n mrcnn_feature_maps = [p2, p3, p4, p5]\n\n rois, rpn_loss_cls, rpn_loss_bbox = self.RCNN_rpn(rpn_feature_maps, im_info, gt_boxes, num_boxes)\n # print(\"rois shape stage1:\",rois.shape)\n # if it is training phrase, then use ground trubut bboxes for refining\n if self.training:\n roi_data = self.RCNN_proposal_target(rois, gt_boxes, num_boxes)\n rois, rois_label, gt_assign, rois_target, rois_inside_ws, rois_outside_ws = roi_data\n\n ## NOTE: additionally, normalize proposals to range [0, 1],\n # this is necessary so that the following roi pooling\n # is correct on different feature maps\n # rois[:, :, 1::2] /= im_info[0][1]\n # rois[:, :, 2::2] /= im_info[0][0]\n\n rois = rois.view(-1, 5)\n rois_label = rois_label.view(-1).long()\n gt_assign = gt_assign.view(-1).long()\n pos_id = rois_label.nonzero().squeeze()\n gt_assign_pos = gt_assign[pos_id]\n rois_label_pos = rois_label[pos_id]\n rois_label_pos_ids = pos_id\n\n rois_pos = Variable(rois[pos_id])\n rois = Variable(rois)\n rois_label = Variable(rois_label)\n\n rois_target = Variable(rois_target.view(-1, rois_target.size(2)))\n rois_inside_ws = Variable(rois_inside_ws.view(-1, rois_inside_ws.size(2)))\n rois_outside_ws = Variable(rois_outside_ws.view(-1, rois_outside_ws.size(2)))\n else:\n ## NOTE: additionally, normalize proposals to range [0, 1],\n # this is necessary so that the following roi pooling\n # is correct on different feature maps\n # rois[:, :, 1::2] /= im_info[0][1]\n # rois[:, :, 2::2] /= im_info[0][0]\n\n rois_label = None\n gt_assign = None\n rois_target = None\n rois_inside_ws = None\n rois_outside_ws = None\n rpn_loss_cls = 0\n rpn_loss_bbox = 0\n rois = rois.view(-1, 5)\n pos_id = torch.arange(0, rois.size(0)).long().type_as(rois).long()\n rois_label_pos_ids = pos_id\n rois_pos = Variable(rois[pos_id])\n rois = Variable(rois)\n\n roi_pool_feat = self._PyramidRoI_Feat(mrcnn_feature_maps, rois, im_info)\n\n # feed pooled features to top model\n pooled_feat = self._head_to_tail(roi_pool_feat)\n\n bbox_pred = self.RCNN_bbox_pred(pooled_feat)\n\n if self.training and not self.class_agnostic:\n # select the corresponding columns according to roi labels\n bbox_pred_view = bbox_pred.view(bbox_pred.size(0), int(bbox_pred.size(1) / 4), 4)\n bbox_pred_select = torch.gather(bbox_pred_view, 1, rois_label.long().view(rois_label.size(0), 1, 1).expand(rois_label.size(0), 1, 4))\n bbox_pred = bbox_pred_select.squeeze(1)\n\n cls_score = self.RCNN_cls_score(pooled_feat)\n cls_prob = F.softmax(cls_score,1)\n # print(cls_prob)\n # print(\"*******************cls prob shape\",cls_prob.shape)\n\n RCNN_loss_cls = 0\n RCNN_loss_bbox = 0\n\n if self.training:\n RCNN_loss_cls = F.cross_entropy(cls_score, rois_label)\n # loss (l1-norm) for bounding box regression\n RCNN_loss_bbox = _smooth_l1_loss(bbox_pred, rois_target, rois_inside_ws, rois_outside_ws)\n\n rois = rois.view(batch_size, -1, rois.size(1))\n bbox_pred = bbox_pred.view(batch_size, -1, bbox_pred.size(1))\n \n if self.training:\n rois_label = rois_label.view(batch_size, -1)\n\n # 2nd-----------------------------\n # decode\n rois = bbox_decode(rois, bbox_pred, batch_size, self.class_agnostic, self.n_classes, im_info, self.training,cls_prob)\n\n if self.training:\n roi_data = self.RCNN_proposal_target(rois, gt_boxes, num_boxes, stage=2)\n rois, rois_label, gt_assign, rois_target, rois_inside_ws, rois_outside_ws = roi_data\n\n rois = rois.view(-1, 5)\n rois_label = rois_label.view(-1).long()\n gt_assign = gt_assign.view(-1).long()\n pos_id = rois_label.nonzero().squeeze()\n gt_assign_pos = gt_assign[pos_id]\n rois_label_pos = rois_label[pos_id]\n rois_label_pos_ids = pos_id\n\n rois_pos = Variable(rois[pos_id])\n rois = Variable(rois)\n rois_label = Variable(rois_label)\n\n rois_target = Variable(rois_target.view(-1, rois_target.size(2)))\n rois_inside_ws = Variable(rois_inside_ws.view(-1, rois_inside_ws.size(2)))\n rois_outside_ws = Variable(rois_outside_ws.view(-1, rois_outside_ws.size(2)))\n else:\n rois_label = None\n gt_assign = None\n rois_target = None\n rois_inside_ws = None\n rois_outside_ws = None\n rpn_loss_cls = 0\n rpn_loss_bbox = 0\n rois = rois.view(-1, 5)\n pos_id = torch.arange(0, rois.size(0)).long().type_as(rois).long()\n # print(pos_id)\n rois_label_pos_ids = pos_id\n rois_pos = Variable(rois[pos_id])\n rois = Variable(rois)\n\n roi_pool_feat = self._PyramidRoI_Feat(mrcnn_feature_maps, rois, im_info)\n # feed pooled features to top model\n pooled_feat = self._head_to_tail_2nd(roi_pool_feat)\n # compute bbox offset\n bbox_pred = self.RCNN_bbox_pred_2nd(pooled_feat)\n if self.training and not self.class_agnostic:\n # select the corresponding columns according to roi labels\n bbox_pred_view = bbox_pred.view(bbox_pred.size(0), int(bbox_pred.size(1) / 4), 4)\n bbox_pred_select = torch.gather(bbox_pred_view, 1,\n rois_label.long().view(rois_label.size(0), 1, 1).expand(rois_label.size(0),\n 1, 4))\n bbox_pred = bbox_pred_select.squeeze(1)\n\n # compute object classification probability\n cls_score = self.RCNN_cls_score_2nd(pooled_feat)\n cls_prob_2nd = F.softmax(cls_score,1) \n\n RCNN_loss_cls_2nd = 0\n RCNN_loss_bbox_2nd = 0\n\n if self.training:\n # loss (cross entropy) for object classification\n RCNN_loss_cls_2nd = F.cross_entropy(cls_score, rois_label)\n # loss (l1-norm) for bounding box regression\n RCNN_loss_bbox_2nd = _smooth_l1_loss(bbox_pred, rois_target, rois_inside_ws, rois_outside_ws)\n\n rois = rois.view(batch_size, -1, rois.size(1))\n # cls_prob_2nd = cls_prob_2nd.view(batch_size, -1, cls_prob_2nd.size(1)) ----------------not be used ---------\n bbox_pred_2nd = bbox_pred.view(batch_size, -1, bbox_pred.size(1))\n\n if self.training:\n rois_label = rois_label.view(batch_size, -1)\n\n # 3rd---------------\n # decode\n rois = bbox_decode(rois, bbox_pred_2nd, batch_size, self.class_agnostic, self.n_classes, im_info, self.training,cls_prob_2nd)\n\n # proposal_target\n # if it is training phrase, then use ground trubut bboxes for refining\n if self.training:\n roi_data = self.RCNN_proposal_target(rois, gt_boxes, num_boxes, stage=3)\n rois, rois_label, gt_assign, rois_target, rois_inside_ws, rois_outside_ws = roi_data\n\n rois = rois.view(-1, 5)\n rois_label = rois_label.view(-1).long()\n gt_assign = gt_assign.view(-1).long()\n pos_id = rois_label.nonzero().squeeze()\n gt_assign_pos = gt_assign[pos_id]\n rois_label_pos = rois_label[pos_id]\n rois_label_pos_ids = pos_id\n\n rois_pos = Variable(rois[pos_id])\n rois = Variable(rois)\n rois_label = Variable(rois_label)\n\n rois_target = Variable(rois_target.view(-1, rois_target.size(2)))\n rois_inside_ws = Variable(rois_inside_ws.view(-1, rois_inside_ws.size(2)))\n rois_outside_ws = Variable(rois_outside_ws.view(-1, rois_outside_ws.size(2)))\n else:\n\n rois_label = None\n gt_assign = None\n rois_target = None\n rois_inside_ws = None\n rois_outside_ws = None\n rpn_loss_cls = 0\n rpn_loss_bbox = 0\n rois = rois.view(-1, 5)\n pos_id = torch.arange(0, rois.size(0)).long().type_as(rois).long()\n rois_label_pos_ids = pos_id\n rois_pos = Variable(rois[pos_id])\n rois = Variable(rois)\n\n roi_pool_feat = self._PyramidRoI_Feat(mrcnn_feature_maps, rois, im_info)\n\n # feed pooled features to top model\n pooled_feat = self._head_to_tail_3rd(roi_pool_feat)\n\n # compute bbox offset\n bbox_pred = self.RCNN_bbox_pred_3rd(pooled_feat)\n if self.training and not self.class_agnostic:\n # select the corresponding columns according to roi labels\n bbox_pred_view = bbox_pred.view(bbox_pred.size(0), int(bbox_pred.size(1) / 4), 4)\n bbox_pred_select = torch.gather(bbox_pred_view, 1,\n rois_label.long().view(rois_label.size(0), 1, 1).expand(\n rois_label.size(0),\n 1, 4))\n bbox_pred = bbox_pred_select.squeeze(1)\n\n # compute object classification probability\n cls_score = self.RCNN_cls_score_3rd(pooled_feat)\n cls_prob_3rd = F.softmax(cls_score, 1)\n\n RCNN_loss_cls_3rd = 0\n RCNN_loss_bbox_3rd = 0\n\n if self.training:\n # loss (cross entropy) for object classification\n RCNN_loss_cls_3rd = F.cross_entropy(cls_score, rois_label)\n # loss (l1-norm) for bounding box regression\n RCNN_loss_bbox_3rd = _smooth_l1_loss(bbox_pred, rois_target, rois_inside_ws, rois_outside_ws)\n\n rois = rois.view(batch_size, -1, rois.size(1))\n cls_prob_3rd = cls_prob_3rd.view(batch_size, -1, cls_prob_3rd.size(1))\n bbox_pred_3rd = bbox_pred.view(batch_size, -1, bbox_pred.size(1))\n\n if self.training:\n rois_label = rois_label.view(batch_size, -1)\n if not self.training:\n # 3rd_avg\n # 1st_3rd\n pooled_feat_1st_3rd = self._head_to_tail(roi_pool_feat)\n cls_score_1st_3rd = self.RCNN_cls_score(pooled_feat_1st_3rd)\n cls_prob_1st_3rd = F.softmax(cls_score_1st_3rd, 1)\n cls_prob_1st_3rd = cls_prob_1st_3rd.view(batch_size, -1, cls_prob_1st_3rd.size(1))\n # 2nd_3rd\n pooled_feat_2nd_3rd = self._head_to_tail_2nd(roi_pool_feat)\n cls_score_2nd_3rd = self.RCNN_cls_score_2nd(pooled_feat_2nd_3rd)\n cls_prob_2nd_3rd = F.softmax(cls_score_2nd_3rd, 1)\n cls_prob_2nd_3rd = cls_prob_2nd_3rd.view(batch_size, -1, cls_prob_2nd_3rd.size(1))\n\n cls_prob_3rd_avg = (cls_prob_1st_3rd + cls_prob_2nd_3rd + cls_prob_3rd) / 3\n else:\n cls_prob_3rd_avg = cls_prob_3rd\n\n return rois, cls_prob_3rd_avg, bbox_pred_3rd, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, RCNN_loss_cls_2nd, RCNN_loss_bbox_2nd, RCNN_loss_cls_3rd, RCNN_loss_bbox_3rd, rois_label\n\n \ndef conv3x3(in_planes, out_planes, stride=1):\n \"3x3 convolution with padding\"\n return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n padding=1, bias=False)\n\n\n# class BasicBlock(nn.Module):\n # expansion = 1\n\n # def __init__(self, inplanes, planes, stride=1, downsample=None):\n # super(BasicBlock, self).__init__()\n # self.conv1 = conv3x3(inplanes, planes, stride)\n # self.bn1 = nn.BatchNorm2d(planes)\n # self.relu = nn.ReLU(inplace=True)\n # self.conv2 = conv3x3(planes, planes)\n # self.bn2 = nn.BatchNorm2d(planes)\n # self.downsample = downsample\n # self.stride = stride\n\n # def forward(self, x):\n # residual = x\n\n # out = self.conv1(x)\n # out = self.bn1(out)\n # out = self.relu(out)\n\n # out = self.conv2(out)\n # out = self.bn2(out)\n\n # if self.downsample is not None:\n # residual = self.downsample(x)\n\n # out += residual\n # out = self.relu(out)\n\n # return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(Bottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, stride=stride, bias=False) # change\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, # change\n padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n self.bn3 = nn.BatchNorm2d(planes * 4)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass ResNet(nn.Module):\n def __init__(self, block, layers, num_classes=1000):\n self.inplanes = 64\n super(ResNet, self).__init__()\n self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n bias=False)\n self.bn1 = nn.BatchNorm2d(64)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=0, ceil_mode=True) # change\n self.layer1 = self._make_layer(block, 64, layers[0])\n self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n self.layer4 = self._make_layer(block, 512, layers[3], stride=2) # different\n self.avgpool = nn.AvgPool2d(7)\n self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n m.weight.data.normal_(0, math.sqrt(2. / n))\n elif isinstance(m, nn.BatchNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n\n def _make_layer(self, block, planes, blocks, stride=1):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(self.inplanes, planes * block.expansion,\n kernel_size=1, stride=stride, bias=False),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n layers.append(block(self.inplanes, planes, stride, downsample))\n self.inplanes = planes * block.expansion\n for i in range(1, blocks):\n layers.append(block(self.inplanes, planes))\n\n return nn.Sequential(*layers)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n\n x = self.avgpool(x)\n x = x.view(x.size(0), -1)\n x = self.fc(x)\n\n return x\n\ndef resnet50(pretrained=False):\n \"\"\"Constructs a ResNet-50 model.\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(Bottleneck, [3, 4, 6, 3])\n if pretrained:\n model.load_state_dict(model_zoo.load_url(model_urls['resnet50']))\n return model\n\ndef resnet101(pretrained=False):\n \"\"\"Constructs a ResNet-101 model.\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(Bottleneck, [3, 4, 23, 3])\n if pretrained:\n model.load_state_dict(model_zoo.load_url(model_urls['resnet101']))\n return model \n \n \nclass resnet(_FPN):\n def __init__(self, classes, num_layers=101, pretrained=False, class_agnostic=False):\n self.dout_base_model = 256\n self.pretrained = pretrained\n self.class_agnostic = class_agnostic\n self.num_layers = num_layers\n \n if num_layers == 101:\n self.model_path = 'data/pretrained_model/resnet101.pth' \n elif num_layers == 50:\n self.model_path = 'data/pretrained_model/resnet50.pth' \n\n _FPN.__init__(self, classes, class_agnostic)\n\n def _init_modules(self):\n if self.num_layers == 101:\n resnet = resnet101()\n elif self.num_layers == 50:\n resnet = resnet50()\n if self.pretrained == True:\n print(\"Loading pretrained weights from %s\" %(self.model_path))\n state_dict = torch.load(self.model_path)\n resnet.load_state_dict({k:v for k,v in state_dict.items() if k in resnet.state_dict()})\n\n self.RCNN_layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool)\n self.RCNN_layer1 = nn.Sequential(resnet.layer1)\n self.RCNN_layer2 = nn.Sequential(resnet.layer2)\n self.RCNN_layer3 = nn.Sequential(resnet.layer3)\n self.RCNN_layer4 = nn.Sequential(resnet.layer4)\n\n # Top layer\n self.RCNN_toplayer = nn.Conv2d(2048, 256, kernel_size=1, stride=1, padding=0) # reduce channel\n\n # Smooth layers\n self.RCNN_smooth1 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)\n self.RCNN_smooth2 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)\n self.RCNN_smooth3 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)\n\n # Lateral layers\n self.RCNN_latlayer1 = nn.Conv2d(1024, 256, kernel_size=1, stride=1, padding=0)\n self.RCNN_latlayer2 = nn.Conv2d( 512, 256, kernel_size=1, stride=1, padding=0)\n self.RCNN_latlayer3 = nn.Conv2d( 256, 256, kernel_size=1, stride=1, padding=0)\n\n # ROI Pool feature downsampling\n self.RCNN_roi_feat_ds = nn.Conv2d(256, 256, kernel_size=3, stride=2, padding=1)\n\n self.RCNN_top = nn.Sequential(\n nn.Conv2d(256, 1024, kernel_size=cfg['POOLING_SIZE'], stride=cfg['POOLING_SIZE'], padding=0),\n nn.ReLU(True),\n nn.Conv2d(1024, 1024, kernel_size=1, stride=1, padding=0),\n nn.ReLU(True)\n )\n\n self.RCNN_top_2nd = nn.Sequential(\n nn.Conv2d(256, 1024, kernel_size=cfg['POOLING_SIZE'], stride=cfg['POOLING_SIZE'], padding=0),\n nn.ReLU(True),\n nn.Conv2d(1024, 1024, kernel_size=1, stride=1, padding=0),\n nn.ReLU(True)\n )\n\n self.RCNN_top_3rd = nn.Sequential(\n nn.Conv2d(256, 1024, kernel_size=cfg['POOLING_SIZE'], stride=cfg['POOLING_SIZE'], padding=0),\n nn.ReLU(True),\n nn.Conv2d(1024, 1024, kernel_size=1, stride=1, padding=0),\n nn.ReLU(True)\n )\n\n self.RCNN_cls_score = nn.Linear(1024, self.n_classes)\n if self.class_agnostic:\n self.RCNN_bbox_pred = nn.Linear(1024, 4)\n else:\n self.RCNN_bbox_pred = nn.Linear(1024, 4 * self.n_classes)\n\n self.RCNN_cls_score_2nd = nn.Linear(1024, self.n_classes)\n if self.class_agnostic:\n self.RCNN_bbox_pred_2nd = nn.Linear(1024, 4)\n else:\n self.RCNN_bbox_pred_2nd = nn.Linear(1024, 4 * self.n_classes)\n\n self.RCNN_cls_score_3rd = nn.Linear(1024, self.n_classes)\n if self.class_agnostic:\n self.RCNN_bbox_pred_3rd = nn.Linear(1024, 4)\n else:\n self.RCNN_bbox_pred_3rd = nn.Linear(1024, 4 * self.n_classes)\n\n # Fix blocks\n for p in self.RCNN_layer0[0].parameters(): p.requires_grad=False\n for p in self.RCNN_layer0[1].parameters(): p.requires_grad=False\n\n if cfg['RESNET_FIXED_BLOCKS'] >= 3:\n for p in self.RCNN_layer3.parameters(): p.requires_grad=False\n if cfg['RESNET_FIXED_BLOCKS'] >= 2:\n for p in self.RCNN_layer2.parameters(): p.requires_grad=False\n if cfg['RESNET_FIXED_BLOCKS'] >= 1:\n for p in self.RCNN_layer1.parameters(): p.requires_grad=False\n\n def set_bn_fix(m):\n classname = m.__class__.__name__\n if classname.find('BatchNorm') != -1:\n for p in m.parameters(): p.requires_grad=False\n\n self.RCNN_layer0.apply(set_bn_fix)\n self.RCNN_layer1.apply(set_bn_fix)\n self.RCNN_layer2.apply(set_bn_fix)\n self.RCNN_layer3.apply(set_bn_fix)\n self.RCNN_layer4.apply(set_bn_fix)\n\n def train(self, mode=True):\n # Override train so that the training mode is set as we want\n nn.Module.train(self, mode)\n if mode:\n # Set fixed blocks to be in eval mode\n self.RCNN_layer0.eval()\n self.RCNN_layer1.eval()\n self.RCNN_layer2.train()\n self.RCNN_layer3.train()\n self.RCNN_layer4.train()\n\n self.RCNN_smooth1.train()\n self.RCNN_smooth2.train()\n self.RCNN_smooth3.train()\n\n self.RCNN_latlayer1.train()\n self.RCNN_latlayer2.train()\n self.RCNN_latlayer3.train()\n\n self.RCNN_toplayer.train()\n\n def set_bn_eval(m):\n classname = m.__class__.__name__\n if classname.find('BatchNorm') != -1:\n m.eval()\n\n self.RCNN_layer0.apply(set_bn_eval)\n self.RCNN_layer1.apply(set_bn_eval)\n self.RCNN_layer2.apply(set_bn_eval)\n self.RCNN_layer3.apply(set_bn_eval)\n self.RCNN_layer4.apply(set_bn_eval)\n\n def _head_to_tail(self, pool5):\n block5 = self.RCNN_top(pool5)\n fc7 = block5.mean(3).mean(2)\n return fc7\n\n def _head_to_tail_2nd(self, pool5):\n block5 = self.RCNN_top_2nd(pool5)\n fc7 = block5.mean(3).mean(2)\n return fc7\n\n def _head_to_tail_3rd(self, pool5):\n block5 = self.RCNN_top_3rd(pool5)\n fc7 = block5.mean(3).mean(2)\n return fc7\n\n#-----------------------------------------------------------------------#\n\nP_img_ext = [\"jpg\", \"png\"]\ndef file_list(path, allfile):\n filelist = os.listdir(path)\n\n for filename in filelist:\n filepath = os.path.join(path, filename)\n if os.path.isdir(filepath):\n file_list(filepath, allfile)\n else:\n if filepath.split(\".\")[-1] in P_img_ext:\n # if filepath.endswith('xml'):\n # allfile.append(filepath[0:-4].strip())\n allfile.append(filepath.strip())\n return allfile\n \n\ndef im_list_to_blob(ims):\n \"\"\"Convert a list of images into a network input.\n\n Assumes images are already prepared (means subtracted, BGR order, ...).\n \"\"\"\n max_shape = np.array([im.shape for im in ims]).max(axis=0)\n num_images = len(ims)\n blob = np.zeros((num_images, max_shape[0], max_shape[1], 3),\n dtype=np.float32)\n for i in range(num_images):\n im = ims[i]\n blob[i, 0:im.shape[0], 0:im.shape[1], :] = im\n\n return blob\n \n \ndef _get_image_blob(im):\n \"\"\"Converts an image into a network input.\n Arguments:\n im (ndarray): a color image in BGR order\n Returns:\n blob (ndarray): a data blob holding an image pyramid\n im_scale_factors (list): list of image scales (relative to im) used\n in the image pyramid\n \"\"\"\n im_orig = im.astype(np.float32, copy=True)\n im_orig -= cfg['PIXEL_MEANS']\n\n im_shape = im_orig.shape\n im_size_min = np.min(im_shape[0:2])\n im_size_max = np.max(im_shape[0:2])\n\n processed_ims = []\n im_scale_factors = []\n\n for target_size in cfg['TEST_SCALES']:\n im_scale = float(target_size) / float(im_size_min)\n # Prevent the biggest axis from being more than MAX_SIZE\n if np.round(im_scale * im_size_max) > cfg['TEST_MAX_SIZE']:\n im_scale = float(cfg['TEST_MAX_SIZE']) / float(im_size_max)\n im = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,\n interpolation=cv2.INTER_LINEAR)\n im_scale_factors.append(im_scale)\n processed_ims.append(im)\n\n # Create a blob to hold the input images\n blob = im_list_to_blob(processed_ims)\n\n return blob, np.array(im_scale_factors)\n\ndef soft_nms(dets, box_scores, sigma=0.5, thresh=0.001, cuda=0):\n \"\"\"\n Build a pytorch implement of Soft NMS algorithm.\n # Augments\n dets: boxes coordinate tensor (format:[y1, x1, y2, x2])\n box_scores: box score tensors\n sigma: variance of Gaussian function\n thresh: score thresh\n cuda: CUDA flag\n # Return\n the index of the selected boxes\n \"\"\"\n\n # Indexes concatenate boxes with the last column\n N = dets.shape[0]\n if cuda:\n indexes = torch.arange(0, N, dtype=torch.float).cuda().view(N, 1)\n else:\n indexes = torch.arange(0, N, dtype=torch.float).view(N, 1)\n dets = torch.cat((dets, indexes), dim=1)\n\n # The order of boxes coordinate is [y1,x1,y2,x2]\n y1 = dets[:, 0]\n x1 = dets[:, 1]\n y2 = dets[:, 2]\n x2 = dets[:, 3]\n scores = box_scores\n areas = (x2 - x1 + 1) * (y2 - y1 + 1)\n\n for i in range(N):\n # intermediate parameters for later parameters exchange\n tscore = scores[i].clone()\n pos = i + 1\n\n if i != N - 1:\n maxscore, maxpos = torch.max(scores[pos:], dim=0)\n if tscore < maxscore:\n dets[i], dets[maxpos.item() + i + 1] = dets[maxpos.item() + i + 1].clone(), dets[i].clone()\n scores[i], scores[maxpos.item() + i + 1] = scores[maxpos.item() + i + 1].clone(), scores[i].clone()\n areas[i], areas[maxpos + i + 1] = areas[maxpos + i + 1].clone(), areas[i].clone()\n\n # IoU calculate\n yy1 = np.maximum(dets[i, 0].to(\"cpu\").numpy(), dets[pos:, 0].to(\"cpu\").numpy())\n xx1 = np.maximum(dets[i, 1].to(\"cpu\").numpy(), dets[pos:, 1].to(\"cpu\").numpy())\n yy2 = np.minimum(dets[i, 2].to(\"cpu\").numpy(), dets[pos:, 2].to(\"cpu\").numpy())\n xx2 = np.minimum(dets[i, 3].to(\"cpu\").numpy(), dets[pos:, 3].to(\"cpu\").numpy())\n \n w = np.maximum(0.0, xx2 - xx1 + 1)\n h = np.maximum(0.0, yy2 - yy1 + 1)\n inter = torch.tensor(w * h).cuda() if cuda else torch.tensor(w * h)\n ovr = torch.div(inter, (areas[i] + areas[pos:] - inter))\n\n # Gaussian decay\n weight = torch.exp(-(ovr * ovr) / sigma)\n scores[pos:] = weight * scores[pos:]\n\n # select the boxes and keep the corresponding indexes\n keep = dets[:, 4][scores > thresh].int()\n\n return keep \n \ndef soft_nms_old( det_proposal, detr_scores, method, thresh, Nt, sigma=0.5 ):\n \n '''\n the soft nms implement using python\n :param dets: the pred_bboxes\n :param method: the policy of decay pred_bbox score in soft nms\n :param thresh: the threshold\n :param Nt: Nt\n :return: the index of pred_bbox after soft nms\n '''\n # print(det_proposal)\n \n x1 = det_proposal[:, 0]\n y1 = det_proposal[:, 1]\n x2 = det_proposal[:, 2]\n y2 = det_proposal[:, 3]\n scores = detr_scores.squeeze(1)\n \n areas = ( y2 - y1 + 1. ) * ( x2 - x1 + 1. ) \n orders = scores.argsort()[::-1]\n keep = []\n \n while orders.size > 0:\n i = orders[0]\n \n keep.append(i)\n for j in orders[1:]:\n xx1 = np.maximum( x1[i], x1[j] )\n yy1 = np.maximum( y1[i], y1[j] )\n xx2 = np.minimum( x2[i], x2[j] )\n yy2 = np.minimum( y2[i], y2[j] )\n w = np.maximum( xx2 - xx1 + 1., 0. )\n h = np.maximum( yy2 - yy1 + 1., 0. )\n \n inter = w * h\n overlap = inter / ( areas[i] + areas[j] - inter )\n\n if method == 1: # linear\n if overlap > Nt:\n weight = 1 - overlap\n else: \n weight = 1 \n elif method == 2: # gaussian\n weight = np.exp( -(overlap * overlap) / sigma )\n else: # original NMS\n if overlap > Nt:\n weight = 0\n else:\n weight = 1\n # print('weight:', weight)\n scores[j] = weight * scores[j]\n # print('scores[j]:', scores[j])\n # print('thresh:', thresh)\n if scores[j] < thresh: \n orders = np.delete( orders, np.where( orders == j ) )\n \n orders = np.delete( orders, 0 ) \n return keep\n \n \ndef vis_detections(im, class_name, dets, thresh=0.0):\n \"\"\"Visual debugging of detections.\"\"\"\n for i in range( dets.shape[0]):\n bbox = tuple(int(np.round(x)) for x in dets[i, :4])\n score = dets[i, -1]\n if score > thresh:\n cv2.rectangle(im, bbox[0:2], bbox[2:4], (0, 204, 0), 4)\n cv2.putText(im, '%s: %.3f' % (class_name, score), (bbox[0], bbox[1] + 15), cv2.FONT_HERSHEY_PLAIN,\n 2.0, (0, 0, 255), thickness=2)\n return im\n\n \n \nif __name__ == '__main__':\n # params \n num_layers = 50\n num_session= 1\n num_epoch = 72\n checkpoint = 1686\n thresh_score_final = 0.05\n thresh_score_final_soft_nms = 0.5\n model_dir =os.path.join(\"models\",\"res\"+str(num_layers),\"pascal_voc\")\n data_dir = '../input/global-wheat-detection/test'\n\n Flag_vis = True\n \n # model name\n load_name = os.path.join(\"../input/mymodels\", 'cascade_fpn_{}_{}_{}.pth'.format(num_session, num_epoch, checkpoint))\n # load_name = 'cascade_fpn_{}_{}_{}.pth'.format(num_session, num_epoch, checkpoint)\n \n # Network\n FPN = resnet(classes, num_layers, pretrained=False, class_agnostic=False)\n FPN.create_architecture()\n\n # load model\n checkpoint = torch.load(load_name)\n FPN.load_state_dict(checkpoint['model'])\n \n # set mode\n FPN.cuda()\n FPN.eval()\n \n print(\"load checkpoin---->\", load_name)\n\n # get test images\n imgs_list= []\n file_list(data_dir, imgs_list)\n \n # commit submission\n submission = []\n for idx, img in enumerate(imgs_list):\n str_print = \"total:{}--currnet:{}--img:{}\".format(len(imgs_list),idx,os.path.basename(img))\n print(img)\n\n # load an image\n im = cv2.imread(img)\n\n \n if Flag_vis: \n img_show = im.copy()\n \n prediction_string = []\n \n # prepare im for forward\n im = im[:,:,::-1] # BGR--->RGB\n blobs, im_scales = _get_image_blob(im)\n im_blob = blobs\n im_info_np = np.array([[im_blob.shape[1], im_blob.shape[2], im_scales[0]]], dtype=np.float32)\n \n im_data_pt = torch.from_numpy(im_blob) # numpy to tensor\n im_data_pt = im_data_pt.permute(0, 3, 1, 2) # NHWC --> NCHW\n im_info_pt = torch.from_numpy(im_info_np) # numpy to tensor\n \n with torch.no_grad():\n im_data = im_data_pt.cuda() \n im_info = im_info_pt.cuda()\n num_boxes = torch.zeros((1), dtype = torch.int64).cuda()\n gt_boxes = torch.zeros((1,1,5), dtype = torch.float32).cuda()\n \n \n # forward\n rois, cls_prob, bbox_pred,rpn_loss_cls, rpn_loss_box, RCNN_loss_cls, RCNN_loss_bbox, RCNN_loss_cls_2nd, RCNN_loss_bbox_2nd, RCNN_loss_cls_3rd, RCNN_loss_bbox_3rd, roi_labels = FPN(im_data, im_info, gt_boxes, num_boxes)\n\n # parse result\n scores = cls_prob.data\n boxes = rois.data[:, :, 1:5]\n \n # box transform\n class_agnostic = False\n args_cuda = True\n if cfg['TEST_BBOX_REG']:\n # Apply bounding-box regression deltas\n box_deltas = bbox_pred.data\n if cfg['TRAIN_BBOX_NORMALIZE_TARGETS_PRECOMPUTED']:\n # Optionally normalize targets by a precomputed mean and stdev\n if class_agnostic:\n if args_cuda > 0:\n box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS']).cuda() + torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS']).cuda()\n else:\n box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS']) + torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS'])\n box_deltas = box_deltas.view(1, -1, 4)\n else:\n if args_cuda > 0:\n box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS']).cuda() + torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS']).cuda()\n else:\n box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_STDS']) + torch.FloatTensor(cfg['TRAIN_BBOX_NORMALIZE_MEANS'])\n box_deltas = box_deltas.view(1, -1, 4 * len(pascal_classes))\n \n pred_boxes = bbox_transform_inv(boxes, box_deltas, 1)\n pred_boxes = clip_boxes(pred_boxes, im_info.data, 1)\n else:\n # Simply repeat the boxes, once for each class\n pred_boxes = np.tile(boxes, (1, scores.shape[1]))\n \n pred_boxes /= im_scales[0] \n\n scores = scores.squeeze()\n pred_boxes = pred_boxes.squeeze()\n\n \n # filter boxes\n for j in range(1, len(pascal_classes)):\n inds = torch.nonzero(scores[:,j]>thresh_score_final).view(-1)\n # if there is det\n if inds.numel() == 0:\n prediction_string.append(\"\")\n else:\n cls_scores = scores[:,j][inds]\n _, order = torch.sort(cls_scores, 0, True)\n if class_agnostic:\n cls_boxes = pred_boxes[inds, :]\n else:\n cls_boxes = pred_boxes[inds][:, j * 4:(j + 1) * 4]\n \n cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1)\n cls_dets = cls_dets[order]\n keep = soft_nms(cls_boxes[order, :], cls_scores[order], thresh=thresh_score_final_soft_nms, cuda=1)\n \n cls_dets = cls_dets[keep.view(-1).long()] \n det_final = cls_dets.cpu().numpy()\n\n\n if det_final.shape[0]==0:\n prediction_string.append(\"\")\n else:\n for i in range( det_final.shape[0]):\n bbox = tuple(int(np.round(x)) for x in det_final[i, :4])\n score = det_final[i, -1]\n x = int(bbox[0])\n y = int(bbox[1])\n w = int(bbox[2]-bbox[0])\n h = int(bbox[3]-bbox[1])\n s = float(score)\n prediction_string.append(\"{} {} {} {} {}\".format(s,x,y,w,h))\n \n if Flag_vis:\n im2show = vis_detections(img_show, pascal_classes[j], det_final)\n \n img_name_save = os.path.basename(img)[:-4]\n prediction_string = \" \".join(prediction_string)\n submission.append([img_name_save,prediction_string])\n \n \n \n #if Flag_vis:\n #result_path = os.path.join(\"results_vis\", os.path.basename(img)) # chagne here\n #cv2.imwrite(result_path, im2show)\n\n\n sample_submission = pd.DataFrame(submission, columns=[\"image_id\",\"PredictionString\"])\n sample_submission.to_csv('submission.csv', index=False)\n\n \n print(\"\\n----------------------END----------------------\")\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}