{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html --use-feature=2020-resolver","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nprint(torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import argparse\nimport logging\nimport math\nimport os\nimport random\nimport shutil\nimport time\nimport logging\nfrom pathlib import Path\nimport subprocess\nimport platform\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom scipy.cluster.vq import kmeans\nfrom scipy.signal import butter, filtfilt\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport glob\nimport cv2\nfrom copy import copy\nimport numpy as np\nimport torch.distributed as dist\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as lr_scheduler\nimport torch.utils.data\nimport yaml\nfrom torch.cuda import amp\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom torch.utils.tensorboard import SummaryWriter\nfrom tqdm import tqdm\nimport math\nimport torchvision.models as models\nfrom copy import deepcopy\nimport torch\nimport torch.nn as nn\n# from torchtools.optim import RangerLars\nimport torch.backends.cudnn as cudnn\nfrom threading import Thread\nfrom torch.utils.data import Dataset\nimport matplotlib.pyplot as plt\nfrom fastai.vision import *\nfrom shutil import copyfile\nimport cv2\nprint(torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **计算loss（在最后部分）**"},{"metadata":{"trusted":true},"cell_type":"code","source":"cudnn.deterministic = True\ncudnn.benchmark = False\ntorch.manual_seed(1)\nclass flat_and_anneal(nn.Module):\n def __init__(self, epochs, anneal_start=0.5, base_lr=0.001, min_lr=0):\n super(flat_and_anneal, self).__init__()\n self.epochs = epochs\n self.anneal_start = anneal_start\n self.base_lr = base_lr\n self.min_lr = min_lr\n\n def forword(self, epoch, optimizer):\n if epoch >= 15:\n epoch = epoch - 15\n for param in optimizer.param_groups:\n lr = self.min_lr + (self.base_lr - self.min_lr) * (1 + math.cos(math.pi * epoch / 5)) / 2\n param['lr'] = lr\ndef get_latest_run(search_dir='./runs'):\n # Return path to most recent 'last.pt' in /runs (i.e. to --resume from)\n last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)\n return max(last_list, key=os.path.getctime) if last_list else ''\n\n\ndef check_git_status():\n # Suggest 'git pull' if repo is out of date\n if platform.system() in ['Linux', 'Darwin'] and not os.path.isfile('/.dockerenv'):\n s = subprocess.check_output('if [ -d .git ]; then git fetch && git status -uno; fi', shell=True).decode('utf-8')\n if 'Your branch is behind' in s:\n print(s[s.find('Your branch is behind'):s.find('\\n\\n')] + '\\n')\n\n\ndef check_img_size(img_size, s=32):\n # Verify img_size is a multiple of stride s\n new_size = make_divisible(img_size, int(s)) # ceil gs-multiple\n if new_size != img_size:\n print('WARNING: --img-size %g must be multiple of max stride %g, updating to %g' % (img_size, s, new_size))\n return new_size\n\n\ndef check_anchors(dataset, model, thr=4.0, imgsz=640):\n # Check anchor fit to data, recompute if necessary\n print('\\nAnalyzing anchors... ', end='')\n m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect()\n shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale\n wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh\n\n def metric(k): # compute metric\n r = wh[:, None] / k[None]\n x = torch.min(r, 1. / r).min(2)[0] # ratio metric\n best = x.max(1)[0] # best_x\n aat = (x > 1. / thr).float().sum(1).mean() # anchors above threshold\n bpr = (best > 1. / thr).float().mean() # best possible recall\n return bpr, aat\n\n bpr, aat = metric(m.anchor_grid.clone().cpu().view(-1, 2))\n print('anchors/target = %.2f, Best Possible Recall (BPR) = %.4f' % (aat, bpr), end='')\n if bpr < 0.98: # threshold to recompute\n print('. Attempting to generate improved anchors, please wait...' % bpr)\n na = m.anchor_grid.numel() // 2 # number of anchors\n new_anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False)\n new_bpr = metric(new_anchors.reshape(-1, 2))[0]\n if new_bpr > bpr: # replace anchors\n new_anchors = torch.tensor(new_anchors, device=m.anchors.device).type_as(m.anchors)\n m.anchor_grid[:] = new_anchors.clone().view_as(m.anchor_grid) # for inference\n m.anchors[:] = new_anchors.clone().view_as(m.anchors) / m.stride.to(m.anchors.device).view(-1, 1, 1) # loss\n check_anchor_order(m)\n print('New anchors saved to model. Update model *.yaml to use these anchors in the future.')\n else:\n print('Original anchors better than new anchors. Proceeding with original anchors.')\n print('') # newline\n\n\ndef check_anchor_order(m):\n # Check anchor order against stride order for YOLOv5 Detect() module m, and correct if necessary\n a = m.anchor_grid.prod(-1).view(-1) # anchor area\n da = a[-1] - a[0] # delta a\n ds = m.stride[-1] - m.stride[0] # delta s\n if da.sign() != ds.sign(): # same order\n print('Reversing anchor order')\n m.anchors[:] = m.anchors.flip(0)\n m.anchor_grid[:] = m.anchor_grid.flip(0)\n\n\ndef check_file(file):\n # Search for file if not found\n if os.path.isfile(file) or file == '':\n return file\n else:\n files = glob.glob('./**/' + file, recursive=True) # find file\n assert len(files), 'File Not Found: %s' % file # assert file was found\n return files[0] # return first file if multiple found\n\n\ndef check_dataset(dict):\n # Download dataset if not found\n val, s = dict.get('val'), dict.get('download')\n if val and len(val):\n val = [os.path.abspath(x) for x in (val if isinstance(val, list) else [val])] # val path\n if not all(os.path.exists(x) for x in val):\n print('\\nWARNING: Dataset not found, nonexistant paths: %s' % [*val])\n if s and len(s): # download script\n print('Downloading %s ...' % s)\n if s.startswith('http') and s.endswith('.zip'): # URL\n f = Path(s).name # filename\n if platform.system() == 'Darwin': # avoid MacOS python requests certificate error\n os.system('curl -L %s -o %s' % (s, f))\n else:\n torch.hub.download_url_to_file(s, f)\n r = os.system('unzip -q %s -d ../ && rm %s' % (f, f)) # unzip\n else: # bash script\n r = os.system(s)\n print('Dataset autodownload %s\\n' % ('success' if r == 0 else 'failure')) # analyze return value\n else:\n raise Exception('Dataset not found.')\n\n\ndef make_divisible(x, divisor):\n # Returns x evenly divisble by divisor\n return math.ceil(x / divisor) * divisor\n\n\ndef labels_to_class_weights(labels, nc=80):\n # Get class weights (inverse frequency) from training labels\n if labels[0] is None: # no labels loaded\n return torch.Tensor()\n\n labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO\n classes = labels[:, 0].astype(np.int) # labels = [class xywh]\n weights = np.bincount(classes, minlength=nc) # occurences per class\n\n # Prepend gridpoint count (for uCE trianing)\n # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image\n # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start\n\n weights[weights == 0] = 1 # replace empty bins with 1\n weights = 1 / weights # number of targets per class\n weights /= weights.sum() # normalize\n return torch.from_numpy(weights)\n\n\ndef labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)):\n # Produces image weights based on class mAPs\n n = len(labels)\n class_counts = np.array([np.bincount(labels[i][:, 0].astype(np.int), minlength=nc) for i in range(n)])\n image_weights = (class_weights.reshape(1, nc) * class_counts).sum(1)\n # index = random.choices(range(n), weights=image_weights, k=1) # weight image sample\n return image_weights\n\n\ndef coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper)\n # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\n # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\\n')\n # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\\n')\n # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco\n # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet\n x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34,\n 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]\n return x\n\n\ndef xyxy2xywh(x):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right\n y = torch.zeros_like(x) if isinstance(x, torch.Tensor) else np.zeros_like(x)\n y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center\n y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center\n y[:, 2] = x[:, 2] - x[:, 0] # width\n y[:, 3] = x[:, 3] - x[:, 1] # height\n return y\n\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = torch.zeros_like(x) if isinstance(x, torch.Tensor) else np.zeros_like(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\n\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\n # Rescale coords (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n coords[:, [0, 2]] -= pad[0] # x padding\n coords[:, [1, 3]] -= pad[1] # y padding\n coords[:, :4] /= gain\n clip_coords(coords, img0_shape)\n return coords\n\n\ndef clip_coords(boxes, img_shape):\n # Clip bounding xyxy bounding boxes to image shape (height, width)\n boxes[:, 0].clamp_(0, img_shape[1]) # x1\n boxes[:, 1].clamp_(0, img_shape[0]) # y1\n boxes[:, 2].clamp_(0, img_shape[1]) # x2\n boxes[:, 3].clamp_(0, img_shape[0]) # y2\n\n\ndef ap_per_class(tp, conf, pred_cls, target_cls):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.\n # Arguments\n tp: True positives (nparray, nx1 or nx10).\n conf: Objectness value from 0-1 (nparray).\n pred_cls: Predicted object classes (nparray).\n target_cls: True object classes (nparray).\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Sort by objectness\n i = np.argsort(-conf)\n tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\n\n # Find unique classes\n unique_classes = np.unique(target_cls)\n\n # Create Precision-Recall curve and compute AP for each class\n pr_score = 0.1 # score to evaluate P and R https://github.com/ultralytics/yolov3/issues/898\n s = [unique_classes.shape[0], tp.shape[1]] # number class, number iou thresholds (i.e. 10 for mAP0.5...0.95)\n ap, p, r = np.zeros(s), np.zeros(s), np.zeros(s)\n for ci, c in enumerate(unique_classes):\n i = pred_cls == c\n n_gt = (target_cls == c).sum() # Number of ground truth objects\n n_p = i.sum() # Number of predicted objects\n\n if n_p == 0 or n_gt == 0:\n continue\n else:\n # Accumulate FPs and TPs\n fpc = (1 - tp[i]).cumsum(0)\n tpc = tp[i].cumsum(0)\n\n # Recall\n recall = tpc / (n_gt + 1e-16) # recall curve\n r[ci] = np.interp(-pr_score, -conf[i], recall[:, 0]) # r at pr_score, negative x, xp because xp decreases\n\n # Precision\n precision = tpc / (tpc + fpc) # precision curve\n p[ci] = np.interp(-pr_score, -conf[i], precision[:, 0]) # p at pr_score\n\n # AP from recall-precision curve\n for j in range(tp.shape[1]):\n ap[ci, j] = compute_ap(recall[:, j], precision[:, j])\n\n # Plot\n # fig, ax = plt.subplots(1, 1, figsize=(5, 5))\n # ax.plot(recall, precision)\n # ax.set_xlabel('Recall')\n # ax.set_ylabel('Precision')\n # ax.set_xlim(0, 1.01)\n # ax.set_ylim(0, 1.01)\n # fig.tight_layout()\n # fig.savefig('PR_curve.png', dpi=300)\n\n # Compute F1 score (harmonic mean of precision and recall)\n f1 = 2 * p * r / (p + r + 1e-16)\n\n return p, r, ap, f1, unique_classes.astype('int32')\n\n\ndef compute_ap(recall, precision):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Source: https://github.com/rbgirshick/py-faster-rcnn.\n # Arguments\n recall: The recall curve (list).\n precision: The precision curve (list).\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Append sentinel values to beginning and end\n mrec = np.concatenate(([0.], recall, [min(recall[-1] + 1E-3, 1.)]))\n mpre = np.concatenate(([0.], precision, [0.]))\n\n # Compute the precision envelope\n mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))\n\n # Integrate area under curve\n method = 'interp' # methods: 'continuous', 'interp'\n if method == 'interp':\n x = np.linspace(0, 1, 101) # 101-point interp (COCO)\n ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate\n else: # 'continuous'\n i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve\n\n return ap\n\n\ndef bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-9):\n # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4\n box2 = box2.T\n\n # Get the coordinates of bounding boxes\n if x1y1x2y2: # x1, y1, x2, y2 = box1\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\n else: # transform from xywh to xyxy\n b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2\n b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2\n b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2\n b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2\n\n # Intersection area\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\n\n # Union Area\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps\n union = w1 * h1 + w2 * h2 - inter + eps\n\n iou = inter / union\n if GIoU or DIoU or CIoU:\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\n if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\n c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 +\n (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared\n if DIoU:\n return iou - rho2 / c2 # DIoU\n elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\n with torch.no_grad():\n alpha = v / ((1 + eps) - iou + v)\n return iou - (rho2 / c2 + v * alpha) # CIoU\n else: # GIoU https://arxiv.org/pdf/1902.09630.pdf\n c_area = cw * ch + eps # convex area\n return iou - (c_area - union) / c_area # GIoU\n else:\n return iou # IoU\n\n\ndef box_iou(box1, box2):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n def box_area(box):\n # box = 4xn\n return (box[2] - box[0]) * (box[3] - box[1])\n\n area1 = box_area(box1.T)\n area2 = box_area(box2.T)\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\n return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)\n\n\ndef wh_iou(wh1, wh2):\n # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2\n wh1 = wh1[:, None] # [N,1,2]\n wh2 = wh2[None] # [1,M,2]\n inter = torch.min(wh1, wh2).prod(2) # [N,M]\n return inter / (wh1.prod(2) + wh2.prod(2) - inter) # iou = inter / (area1 + area2 - inter)\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super(FocalLoss, self).__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = 'none' # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == 'mean':\n return loss.mean()\n elif self.reduction == 'sum':\n return loss.sum()\n else: # 'none'\n return loss\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass BCEBlurWithLogitsLoss(nn.Module):\n # BCEwithLogitLoss() with reduced missing label effects.\n def __init__(self, alpha=0.05):\n super(BCEBlurWithLogitsLoss, self).__init__()\n self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss()\n self.alpha = alpha\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n pred = torch.sigmoid(pred) # prob from logits\n dx = pred - true # reduce only missing label effects\n # dx = (pred - true).abs() # reduce missing label and false label effects\n alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4))\n loss *= alpha_factor\n return loss.mean()\n\n\n'''计算损失'''\ndef compute_loss(p, targets, model): # predictions, targets, model\n '''\n\n :param p: 【bs*anchor*h*w*(C+5)，。。。，。。。】一共有三个detect的输出\n :param targets: 所有图片的真实方框标签（b,class,x,y,w,h)，其中b时在该batch的序号\n :param model:\n :return:\n '''\n device = targets.device\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\n\n '''build_targets返回的值'''\n '''\n tcls:返回[bs*nl0,bs*nl1,bs*nl2],所谓nlj就是第j个detect层的anchor数量，这里就是返回每个anchor对应的target的标签\n tbox:返回[bs*nl0*4,bs*nl1*4,bs*nl2*4]返回第j个detect层的anchor对应的target的坐标,由于每一个detect层特征图大小不一样，故对应target\n 坐标应该不一样\n \n indices:[bs*nl0*4,bs*nl1*4,bs*nl2*4]其中4是(b,a,gj,gi)\n \n b:b就是图片在batch中的讯号\n a:a就是anchor在nlj个anchor中的序号\n gj,gi：就是anchor对应特征图grid的坐标\n \n anchors:保存的是anchor的wh值\n \n '''\n tcls, tbox, indices, anchors = build_targets(p, targets, model) # targets\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['cls_pw']])).to(device)\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['obj_pw']])).to(device)\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n cp, cn = smooth_BCE(eps=0.0)\n\n # Focal loss\n g = h['fl_gamma'] # focal loss gamma\n if g > 0:\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\n\n # Losses\n nt = 0 # number of targets\n np = len(p) # number of outputs\n balance = [4.0, 1.0, 0.4] if np == 3 else [4.0, 1.0, 0.4, 0.1] # P3-5 or P3-6\n for i, pi in enumerate(p): # layer index, layer predictions\n b, a, gj, gi = indices[i] # image, anchor, gridy, gridx\n\n '''获取第i个detect的输出'''\n tobj = torch.zeros_like(pi[..., 0], device=device) # target obj\n\n n = b.shape[0] # number of targets\n if n:\n '''计算target的总数量'''\n nt += n # cumulative targets\n\n '''选择target对应的第i层的anchor的坐标'''\n ps = pi[b, a, gj, gi] # prediction subset corresponding to targets\n\n # Regression\n '''根据公式进行修正，计算anchor真正的预测值'''\n pxy = ps[:, :2].sigmoid() * 2. - 0.5\n pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]\n pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box\n\n '''计算预测box和真实框的iou，然后使用iouloss'''\n giou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # giou(prediction, target)\n lbox += (1.0 - giou).mean() # giou loss\n\n # Objectness\n '''根据giou判断anchor的置信度'''\n tobj[b, a, gj, gi] = (1.0 - model.gr) + model.gr * giou.detach().clamp(0).type(tobj.dtype) # giou ratio\n\n # Classification\n if model.nc > 1: # cls loss (only if multiple classes)\n '''前面5个是每个anchor的置信度和xywh，不需要提取，我们只提取每个anchor的类别预测'''\n t = torch.full_like(ps[:, 5:], cn, device=device) # targets\n t[range(n), tcls[i]] = cp\n '''计算anchor类别损失'''\n lcls += BCEcls(ps[:, 5:], t) # BCE\n\n # Append targets to text file\n # with open('targets.txt', 'a') as file:\n # [file.write('%11.5g ' * 4 % tuple(x) + '\\n') for x in torch.cat((txy[i], twh[i]), 1)]\n '''计算置信度损失'''\n lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss\n\n s = 3 / np # output count scaling\n lbox *= h['giou'] * s\n lobj *= h['obj'] * s * (1.4 if np == 4 else 1.)\n lcls *= h['cls'] * s\n bs = tobj.shape[0] # batch size\n\n loss = lbox + lobj + lcls\n return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()\n\n\n'''找到与target对应的anchor的位置'''\ndef build_targets(p, targets, model):\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\n det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module\n na, nt = det.na, targets.shape[0] # number of anchors, targets\n tcls, tbox, indices, anch = [], [], [], []\n gain = torch.ones(7, device=targets.device) # normalized to gridspace gain\n ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\n\n '''建立每个target对应每个anchor的指示器'''\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices\n\n g = 0.5 # bias\n off = torch.tensor([[0, 0],\n [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\n ], device=targets.device).float() * g # offsets\n\n for i in range(det.nl):\n anchors = det.anchors[i]\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\n\n # Match targets to anchors\n t = targets * gain\n if nt:\n # Matches\n r = t[:, :, 4:6] / anchors[:, None] # wh ratio\n j = torch.max(r, 1. / r).max(2)[0] < model.hyp['anchor_t'] # compare\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\n t = t[j] # filter\n\n # Offsets\n gxy = t[:, 2:4] # grid xy\n gxi = gain[[2, 3]] - gxy # inverse\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\n l, m = ((gxi % 1. < g) & (gxi > 1.)).T\n j = torch.stack((torch.ones_like(j), j, k, l, m))\n t = t.repeat((5, 1, 1))[j]\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\n else:\n t = targets[0]\n offsets = 0\n\n # Define\n b, c = t[:, :2].long().T # image, class\n gxy = t[:, 2:4] # grid xy\n gwh = t[:, 4:6] # grid wh\n gij = (gxy - offsets).long()\n gi, gj = gij.T # grid xy indices\n\n # Append\n a = t[:, 6].long() # anchor indices\n indices.append((b, a, gj, gi)) # image, anchor, grid indices\n tbox.append(torch.cat((gxy - gij, gwh), 1)) # box\n anch.append(anchors[a]) # anchors\n tcls.append(c) # class\n\n return tcls, tbox, indices, anch\n\n\ndef non_max_suppression(prediction, conf_thres=0.1, iou_thres=0.6, merge=False, classes=None, agnostic=False):\n \"\"\"Performs Non-Maximum Suppression (NMS) on inference results\n\n Returns:\n detections with shape: nx6 (x1, y1, x2, y2, conf, cls)\n \"\"\"\n\n nc = prediction[0].shape[1] - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n\n # Settings\n min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height\n max_det = 300 # maximum number of detections per image\n time_limit = 10.0 # seconds to quit after\n redundant = True # require redundant detections\n multi_label = nc > 1 # multiple labels per box (adds 0.5ms/img)\n\n t = time.time()\n output = [None] * prediction.shape[0]\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\n else: # best class only\n conf, j = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # If none remain process next image\n n = x.shape[0] # number of boxes\n if not n:\n continue\n\n # Sort by confidence\n # x = x[x[:, 4].argsort(descending=True)]\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torch.ops.torchvision.nms(boxes, scores, iou_thres)\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n try: # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n except: # possible CUDA error https://github.com/ultralytics/yolov3/issues/1139\n print(x, i, x.shape, i.shape)\n pass\n\n output[xi] = x[i]\n if (time.time() - t) > time_limit:\n break # time limit exceeded\n\n return output\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef strip_optimizer(f='weights/best.pt', s=''): # from utils.general import *; strip_optimizer()\n # Strip optimizer from 'f' to finalize training, optionally save as 's'\n x = torch.load(f, map_location=torch.device('cpu'))\n x['optimizer'] = None\n x['training_results'] = None\n x['epoch'] = -1\n x['model'].half() # to FP16\n for p in x['model'].parameters():\n p.requires_grad = False\n torch.save(x, s or f)\n mb = os.path.getsize(s or f) / 1E6 # filesize\n print('Optimizer stripped from %s,%s %.1fMB' % (f, (' saved as %s,' % s) if s else '', mb))\n\n\ndef coco_class_count(path='../coco/labels/train2014/'):\n # Histogram of occurrences per class\n nc = 80 # number classes\n x = np.zeros(nc, dtype='int32')\n files = sorted(glob.glob('%s/*.*' % path))\n for i, file in enumerate(files):\n labels = np.loadtxt(file, dtype=np.float32).reshape(-1, 5)\n x += np.bincount(labels[:, 0].astype('int32'), minlength=nc)\n print(i, len(files))\n\n\ndef coco_only_people(path='../coco/labels/train2017/'): # from utils.general import *; coco_only_people()\n # Find images with only people\n files = sorted(glob.glob('%s/*.*' % path))\n for i, file in enumerate(files):\n labels = np.loadtxt(file, dtype=np.float32).reshape(-1, 5)\n if all(labels[:, 0] == 0):\n print(labels.shape[0], file)\n\n\ndef crop_images_random(path='../images/', scale=0.50): # from utils.general import *; crop_images_random()\n # crops images into random squares up to scale fraction\n # WARNING: overwrites images!\n for file in tqdm(sorted(glob.glob('%s/*.*' % path))):\n img = cv2.imread(file) # BGR\n if img is not None:\n h, w = img.shape[:2]\n\n # create random mask\n a = 30 # minimum size (pixels)\n mask_h = random.randint(a, int(max(a, h * scale))) # mask height\n mask_w = mask_h # mask width\n\n # box\n xmin = max(0, random.randint(0, w) - mask_w // 2)\n ymin = max(0, random.randint(0, h) - mask_h // 2)\n xmax = min(w, xmin + mask_w)\n ymax = min(h, ymin + mask_h)\n\n # apply random color mask\n cv2.imwrite(file, img[ymin:ymax, xmin:xmax])\n\n\ndef coco_single_class_labels(path='../coco/labels/train2014/', label_class=43):\n # Makes single-class coco datasets. from utils.general import *; coco_single_class_labels()\n if os.path.exists('new/'):\n shutil.rmtree('new/') # delete output folder\n os.makedirs('new/') # make new output folder\n os.makedirs('new/labels/')\n os.makedirs('new/images/')\n for file in tqdm(sorted(glob.glob('%s/*.*' % path))):\n with open(file, 'r') as f:\n labels = np.array([x.split() for x in f.read().splitlines()], dtype=np.float32)\n i = labels[:, 0] == label_class\n if any(i):\n img_file = file.replace('labels', 'images').replace('txt', 'jpg')\n labels[:, 0] = 0 # reset class to 0\n with open('new/images.txt', 'a') as f: # add image to dataset list\n f.write(img_file + '\\n')\n with open('new/labels/' + Path(file).name, 'a') as f: # write label\n for l in labels[i]:\n f.write('%g %.6f %.6f %.6f %.6f\\n' % tuple(l))\n shutil.copyfile(src=img_file, dst='new/images/' + Path(file).name.replace('txt', 'jpg')) # copy images\n\n\ndef kmean_anchors(path='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True):\n \"\"\" Creates kmeans-evolved anchors from training dataset\n\n Arguments:\n path: path to dataset *.yaml, or a loaded dataset\n n: number of anchors\n img_size: image size used for training\n thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0\n gen: generations to evolve anchors using genetic algorithm\n\n Return:\n k: kmeans evolved anchors\n\n Usage:\n from utils.general import *; _ = kmean_anchors()\n \"\"\"\n thr = 1. / thr\n\n def metric(k, wh): # compute metrics\n r = wh[:, None] / k[None]\n x = torch.min(r, 1. / r).min(2)[0] # ratio metric\n # x = wh_iou(wh, torch.tensor(k)) # iou metric\n return x, x.max(1)[0] # x, best_x\n\n def fitness(k): # mutation fitness\n _, best = metric(torch.tensor(k, dtype=torch.float32), wh)\n return (best * (best > thr).float()).mean() # fitness\n\n def print_results(k):\n k = k[np.argsort(k.prod(1))] # sort small to large\n x, best = metric(k, wh0)\n bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr\n print('thr=%.2f: %.4f best possible recall, %.2f anchors past thr' % (thr, bpr, aat))\n print('n=%g, img_size=%s, metric_all=%.3f/%.3f-mean/best, past_thr=%.3f-mean: ' %\n (n, img_size, x.mean(), best.mean(), x[x > thr].mean()), end='')\n for i, x in enumerate(k):\n print('%i,%i' % (round(x[0]), round(x[1])), end=', ' if i < len(k) - 1 else '\\n') # use in *.cfg\n return k\n\n if isinstance(path, str): # *.yaml file\n with open(path) as f:\n data_dict = yaml.load(f, Loader=yaml.FullLoader) # model dict\n\n dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True)\n else:\n dataset = path # dataset\n\n # Get label wh\n shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh\n\n # Filter\n i = (wh0 < 3.0).any(1).sum()\n if i:\n print('WARNING: Extremely small objects found. '\n '%g of %g labels are < 3 pixels in width or height.' % (i, len(wh0)))\n wh = wh0[(wh0 >= 2.0).any(1)] # filter > 2 pixels\n\n # Kmeans calculation\n print('Running kmeans for %g anchors on %g points...' % (n, len(wh)))\n s = wh.std(0) # sigmas for whitening\n k, dist = kmeans(wh / s, n, iter=30) # points, mean distance\n k *= s\n wh = torch.tensor(wh, dtype=torch.float32) # filtered\n wh0 = torch.tensor(wh0, dtype=torch.float32) # unflitered\n k = print_results(k)\n\n # Plot\n # k, d = [None] * 20, [None] * 20\n # for i in tqdm(range(1, 21)):\n # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7))\n # ax = ax.ravel()\n # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.')\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh\n # ax[0].hist(wh[wh[:, 0]<100, 0],400)\n # ax[1].hist(wh[wh[:, 1]<100, 1],400)\n # fig.tight_layout()\n # fig.savefig('wh.png', dpi=200)\n\n # Evolve\n npr = np.random\n f, sh, mp, s = fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma\n pbar = tqdm(range(gen), desc='Evolving anchors with Genetic Algorithm') # progress bar\n for _ in pbar:\n v = np.ones(sh)\n while (v == 1).all(): # mutate until a change occurs (prevent duplicates)\n v = ((npr.random(sh) < mp) * npr.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0)\n kg = (k.copy() * v).clip(min=2.0)\n fg = fitness(kg)\n if fg > f:\n f, k = fg, kg.copy()\n pbar.desc = 'Evolving anchors with Genetic Algorithm: fitness = %.4f' % f\n if verbose:\n print_results(k)\n\n return print_results(k)\n\n\ndef print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''):\n # Print mutation results to evolve.txt (for use with train.py --evolve)\n a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys\n b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values\n c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3)\n print('\\n%s\\n%s\\nEvolved fitness: %s\\n' % (a, b, c))\n\n if bucket:\n os.system('gsutil cp gs://%s/evolve.txt .' % bucket) # download evolve.txt\n\n with open('evolve.txt', 'a') as f: # append result\n f.write(c + b + '\\n')\n x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0) # load unique rows\n x = x[np.argsort(-fitness(x))] # sort\n np.savetxt('evolve.txt', x, '%10.3g') # save sort by fitness\n\n # Save yaml\n for i, k in enumerate(hyp.keys()):\n hyp[k] = float(x[0, i + 7])\n with open(yaml_file, 'w') as f:\n results = tuple(x[0, :7])\n c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3)\n f.write('# Hyperparameter Evolution Results\\n# Generations: %g\\n# Metrics: ' % len(x) + c + '\\n\\n')\n yaml.dump(hyp, f, sort_keys=False)\n\n if bucket:\n os.system('gsutil cp evolve.txt %s gs://%s' % (yaml_file, bucket)) # upload\n\n\ndef apply_classifier(x, model, img, im0):\n # applies a second stage classifier to yolo outputs\n im0 = [im0] if isinstance(im0, np.ndarray) else im0\n for i, d in enumerate(x): # per image\n if d is not None and len(d):\n d = d.clone()\n\n # Reshape and pad cutouts\n b = xyxy2xywh(d[:, :4]) # boxes\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square\n b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad\n d[:, :4] = xywh2xyxy(b).long()\n\n # Rescale boxes from img_size to im0 size\n scale_coords(img.shape[2:], d[:, :4], im0[i].shape)\n\n # Classes\n pred_cls1 = d[:, 5].long()\n ims = []\n for j, a in enumerate(d): # per item\n cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])]\n im = cv2.resize(cutout, (224, 224)) # BGR\n # cv2.imwrite('test%i.jpg' % j, cutout)\n\n im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32\n im /= 255.0 # 0 - 255 to 0.0 - 1.0\n ims.append(im)\n\n pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction\n x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections\n\n return x\n\n\ndef fitness(x):\n # Returns fitness (for use with results.txt or evolve.txt)\n w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef output_to_target(output, width, height):\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf]\n if isinstance(output, torch.Tensor):\n output = output.cpu().numpy()\n\n targets = []\n for i, o in enumerate(output):\n if o is not None:\n for pred in o:\n box = pred[:4]\n w = (box[2] - box[0]) / width\n h = (box[3] - box[1]) / height\n x = box[0] / width + w / 2\n y = box[1] / height + h / 2\n conf = pred[4]\n cls = int(pred[5])\n\n targets.append([i, cls, x, y, w, h, conf])\n\n return np.array(targets)\n\n\ndef increment_dir(dir, comment=''):\n # Increments a directory runs/exp1 --> runs/exp2_comment\n n = 0 # number\n dir = str(Path(dir)) # os-agnostic\n d = sorted(glob.glob(dir + '*')) # directories\n if len(d):\n n = max([int(x[len(dir):x.find('_') if '_' in x else None]) for x in d]) + 1 # increment\n return dir + str(n) + ('_' + comment if comment else '')\n\n\n# Plotting functions ---------------------------------------------------------------------------------------------------\ndef hist2d(x, y, n=100):\n # 2d histogram used in labels.png and evolve.png\n xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n)\n hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges))\n xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1)\n yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1)\n return np.log(hist[xidx, yidx])\n\n\ndef butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5):\n # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy\n def butter_lowpass(cutoff, fs, order):\n nyq = 0.5 * fs\n normal_cutoff = cutoff / nyq\n b, a = butter(order, normal_cutoff, btype='low', analog=False)\n return b, a\n\n b, a = butter_lowpass(cutoff, fs, order=order)\n return filtfilt(b, a, data) # forward-backward filter\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n # Plots one bounding box on image img\n tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness\n color = color or [random.randint(0, 255) for _ in range(3)]\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n if label:\n tf = max(tl - 1, 1) # font thickness\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\n\ndef plot_wh_methods(): # from utils.general import *; plot_wh_methods()\n # Compares the two methods for width-height anchor multiplication\n # https://github.com/ultralytics/yolov3/issues/168\n x = np.arange(-4.0, 4.0, .1)\n ya = np.exp(x)\n yb = torch.sigmoid(torch.from_numpy(x)).numpy() * 2\n\n fig = plt.figure(figsize=(6, 3), dpi=150)\n plt.plot(x, ya, '.-', label='YOLOv3')\n plt.plot(x, yb ** 2, '.-', label='YOLOv5 ^2')\n plt.plot(x, yb ** 1.6, '.-', label='YOLOv5 ^1.6')\n plt.xlim(left=-4, right=4)\n plt.ylim(bottom=0, top=6)\n plt.xlabel('input')\n plt.ylabel('output')\n plt.grid()\n plt.legend()\n fig.tight_layout()\n fig.savefig('comparison.png', dpi=200)\n\n\ndef plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16):\n tl = 3 # line thickness\n tf = max(tl - 1, 1) # font thickness\n if os.path.isfile(fname): # do not overwrite\n return None\n\n if isinstance(images, torch.Tensor):\n images = images.cpu().float().numpy()\n\n if isinstance(targets, torch.Tensor):\n targets = targets.cpu().numpy()\n\n # un-normalise\n if np.max(images[0]) <= 1:\n images *= 255\n\n bs, _, h, w = images.shape # batch size, _, height, width\n bs = min(bs, max_subplots) # limit plot images\n ns = np.ceil(bs ** 0.5) # number of subplots (square)\n\n # Check if we should resize\n scale_factor = max_size / max(h, w)\n if scale_factor < 1:\n h = math.ceil(scale_factor * h)\n w = math.ceil(scale_factor * w)\n\n # Empty array for output\n mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8)\n\n # Fix class - colour map\n prop_cycle = plt.rcParams['axes.prop_cycle']\n # https://stackoverflow.com/questions/51350872/python-from-color-name-to-rgb\n hex2rgb = lambda h: tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\n color_lut = [hex2rgb(h) for h in prop_cycle.by_key()['color']]\n\n for i, img in enumerate(images):\n if i == max_subplots: # if last batch has fewer images than we expect\n break\n\n block_x = int(w * (i // ns))\n block_y = int(h * (i % ns))\n\n img = img.transpose(1, 2, 0)\n if scale_factor < 1:\n img = cv2.resize(img, (w, h))\n\n mosaic[block_y:block_y + h, block_x:block_x + w, :] = img\n if len(targets) > 0:\n image_targets = targets[targets[:, 0] == i]\n boxes = xywh2xyxy(image_targets[:, 2:6]).T\n classes = image_targets[:, 1].astype('int')\n gt = image_targets.shape[1] == 6 # ground truth if no conf column\n conf = None if gt else image_targets[:, 6] # check for confidence presence (gt vs pred)\n\n boxes[[0, 2]] *= w\n boxes[[0, 2]] += block_x\n boxes[[1, 3]] *= h\n boxes[[1, 3]] += block_y\n for j, box in enumerate(boxes.T):\n cls = int(classes[j])\n color = color_lut[cls % len(color_lut)]\n cls = names[cls] if names else cls\n if gt or conf[j] > 0.3: # 0.3 conf thresh\n label = '%s' % cls if gt else '%s %.1f' % (cls, conf[j])\n plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl)\n\n # Draw image filename labels\n if paths is not None:\n label = os.path.basename(paths[i])[:40] # trim to 40 char\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf,\n lineType=cv2.LINE_AA)\n\n # Image border\n cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3)\n\n if fname is not None:\n mosaic = cv2.resize(mosaic, (int(ns * w * 0.5), int(ns * h * 0.5)), interpolation=cv2.INTER_AREA)\n cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB))\n\n return mosaic\n\n\ndef plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''):\n # Plot LR simulating training for full epochs\n optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals\n y = []\n for _ in range(epochs):\n scheduler.step()\n y.append(optimizer.param_groups[0]['lr'])\n plt.plot(y, '.-', label='LR')\n plt.xlabel('epoch')\n plt.ylabel('LR')\n plt.grid()\n plt.xlim(0, epochs)\n plt.ylim(0)\n plt.tight_layout()\n plt.savefig(Path(save_dir) / 'LR.png', dpi=200)\n\n\ndef plot_test_txt(): # from utils.general import *; plot_test()\n # Plot test.txt histograms\n x = np.loadtxt('test.txt', dtype=np.float32)\n box = xyxy2xywh(x[:, :4])\n cx, cy = box[:, 0], box[:, 1]\n\n fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True)\n ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0)\n ax.set_aspect('equal')\n plt.savefig('hist2d.png', dpi=300)\n\n fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True)\n ax[0].hist(cx, bins=600)\n ax[1].hist(cy, bins=600)\n plt.savefig('hist1d.png', dpi=200)\n\n\ndef plot_targets_txt(): # from utils.general import *; plot_targets_txt()\n # Plot targets.txt histograms\n x = np.loadtxt('targets.txt', dtype=np.float32).T\n s = ['x targets', 'y targets', 'width targets', 'height targets']\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\n ax = ax.ravel()\n for i in range(4):\n ax[i].hist(x[i], bins=100, label='%.3g +/- %.3g' % (x[i].mean(), x[i].std()))\n ax[i].legend()\n ax[i].set_title(s[i])\n plt.savefig('targets.jpg', dpi=200)\n\n\ndef plot_study_txt(f='study.txt', x=None): # from utils.general import *; plot_study_txt()\n # Plot study.txt generated by test.py\n fig, ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True)\n ax = ax.ravel()\n\n fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True)\n for f in ['study/study_coco_yolov5%s.txt' % x for x in ['s', 'm', 'l', 'x']]:\n y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T\n x = np.arange(y.shape[1]) if x is None else np.array(x)\n s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_inference (ms/img)', 't_NMS (ms/img)', 't_total (ms/img)']\n for i in range(7):\n ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8)\n ax[i].set_title(s[i])\n\n j = y[3].argmax() + 1\n ax2.plot(y[6, :j], y[3, :j] * 1E2, '.-', linewidth=2, markersize=8,\n label=Path(f).stem.replace('study_coco_', '').replace('yolo', 'YOLO'))\n\n ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5],\n 'k.-', linewidth=2, markersize=8, alpha=.25, label='EfficientDet')\n\n ax2.grid()\n ax2.set_xlim(0, 30)\n ax2.set_ylim(28, 50)\n ax2.set_yticks(np.arange(30, 55, 5))\n ax2.set_xlabel('GPU Speed (ms/img)')\n ax2.set_ylabel('COCO AP val')\n ax2.legend(loc='lower right')\n plt.savefig('study_mAP_latency.png', dpi=300)\n plt.savefig(f.replace('.txt', '.png'), dpi=300)\n\n\ndef plot_labels(labels, save_dir=''):\n # plot dataset labels\n c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes\n nc = int(c.max() + 1) # number of classes\n\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\n ax = ax.ravel()\n ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8)\n ax[0].set_xlabel('classes')\n ax[1].scatter(b[0], b[1], c=hist2d(b[0], b[1], 90), cmap='jet')\n ax[1].set_xlabel('x')\n ax[1].set_ylabel('y')\n ax[2].scatter(b[2], b[3], c=hist2d(b[2], b[3], 90), cmap='jet')\n ax[2].set_xlabel('width')\n ax[2].set_ylabel('height')\n plt.savefig(Path(save_dir) / 'labels.png', dpi=200)\n plt.close()\n\n\ndef plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.general import *; plot_evolution()\n # Plot hyperparameter evolution results in evolve.txt\n with open(yaml_file) as f:\n hyp = yaml.load(f, Loader=yaml.FullLoader)\n x = np.loadtxt('evolve.txt', ndmin=2)\n f = fitness(x)\n # weights = (f - f.min()) ** 2 # for weighted results\n plt.figure(figsize=(10, 10), tight_layout=True)\n matplotlib.rc('font', **{'size': 8})\n for i, (k, v) in enumerate(hyp.items()):\n y = x[:, i + 7]\n # mu = (y * weights).sum() / weights.sum() # best weighted result\n mu = y[f.argmax()] # best single result\n plt.subplot(5, 5, i + 1)\n plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none')\n plt.plot(mu, f.max(), 'k+', markersize=15)\n plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters\n if i % 5 != 0:\n plt.yticks([])\n print('%15s: %.3g' % (k, mu))\n plt.savefig('evolve.png', dpi=200)\n print('\\nPlot saved as evolve.png')\n\n\ndef plot_results_overlay(start=0, stop=0): # from utils.general import *; plot_results_overlay()\n # Plot training 'results*.txt', overlaying train and val losses\n s = ['train', 'train', 'train', 'Precision', 'mAP@0.5', 'val', 'val', 'val', 'Recall', 'mAP@0.5:0.95'] # legends\n t = ['GIoU', 'Objectness', 'Classification', 'P-R', 'mAP-F1'] # titles\n for f in sorted(glob.glob('results*.txt') + glob.glob('../../Downloads/results*.txt')):\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\n n = results.shape[1] # number of rows\n x = range(start, min(stop, n) if stop else n)\n fig, ax = plt.subplots(1, 5, figsize=(14, 3.5), tight_layout=True)\n ax = ax.ravel()\n for i in range(5):\n for j in [i, i + 5]:\n y = results[j, x]\n ax[i].plot(x, y, marker='.', label=s[j])\n # y_smooth = butter_lowpass_filtfilt(y)\n # ax[i].plot(x, np.gradient(y_smooth), marker='.', label=s[j])\n\n ax[i].set_title(t[i])\n ax[i].legend()\n ax[i].set_ylabel(f) if i == 0 else None # add filename\n fig.savefig(f.replace('.txt', '.png'), dpi=200)\n\n\ndef plot_results(start=0, stop=0, bucket='', id=(), labels=(),\n save_dir=''): # from utils.general import *; plot_results()\n # Plot training 'results*.txt' as seen in https://github.com/ultralytics/yolov5#reproduce-our-training\n fig, ax = plt.subplots(2, 5, figsize=(12, 6))\n ax = ax.ravel()\n s = ['GIoU', 'Objectness', 'Classification', 'Precision', 'Recall',\n 'val GIoU', 'val Objectness', 'val Classification', 'mAP@0.5', 'mAP@0.5:0.95']\n if bucket:\n # os.system('rm -rf storage.googleapis.com')\n # files = ['https://storage.googleapis.com/%s/results%g.txt' % (bucket, x) for x in id]\n files = ['results%g.txt' % x for x in id]\n c = ('gsutil cp ' + '%s ' * len(files) + '.') % tuple('gs://%s/results%g.txt' % (bucket, x) for x in id)\n os.system(c)\n else:\n files = glob.glob(str(Path(save_dir) / 'results*.txt')) + glob.glob('../../Downloads/results*.txt')\n for fi, f in enumerate(files):\n try:\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\n n = results.shape[1] # number of rows\n x = range(start, min(stop, n) if stop else n)\n for i in range(10):\n y = results[i, x]\n if i in [0, 1, 2, 5, 6, 7]:\n y[y == 0] = np.nan # dont show zero loss values\n # y /= y[0] # normalize\n label = labels[fi] if len(labels) else Path(f).stem\n ax[i].plot(x, y, marker='.', label=label, linewidth=1, markersize=6)\n ax[i].set_title(s[i])\n # if i in [5, 6, 7]: # share train and val loss y axes\n # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])\n except Exception as e:\n print('Warning: Plotting error for %s; %s' % (f, e))\n\n fig.tight_layout()\n ax[1].legend()\n fig.savefig(Path(save_dir) / 'results.png', dpi=200)\n\n\ndef time_synchronized():\n torch.cuda.synchronize() if torch.cuda.is_available() else None\n return time.time()\n\n\ndef is_parallel(model):\n return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)\n\n\ndef intersect_dicts(da, db, exclude=()):\n # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values\n return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape}\n\n\ndef initialize_weights(model):\n for m in model.modules():\n t = type(m)\n if t is nn.Conv2d:\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n elif t is nn.BatchNorm2d:\n m.eps = 1e-3\n m.momentum = 0.03\n elif t in [nn.LeakyReLU, nn.ReLU, nn.ReLU6]:\n m.inplace = True\n\n\ndef find_modules(model, mclass=nn.Conv2d):\n # Finds layer indices matching module class 'mclass'\n return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]\n\n\ndef sparsity(model):\n # Return global model sparsity\n a, b = 0., 0.\n for p in model.parameters():\n a += p.numel()\n b += (p == 0).sum()\n return b / a\n\n\ndef prune(model, amount=0.3):\n # Prune model to requested global sparsity\n import torch.nn.utils.prune as prune\n print('Pruning model... ', end='')\n for name, m in model.named_modules():\n if isinstance(m, nn.Conv2d):\n prune.l1_unstructured(m, name='weight', amount=amount) # prune\n prune.remove(m, 'weight') # make permanent\n print(' %.3g global sparsity' % sparsity(model))\n\n\ndef fuse_conv_and_bn(conv, bn):\n # https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n with torch.no_grad():\n # init\n fusedconv = nn.Conv2d(conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n bias=True).to(conv.weight.device)\n\n # prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.size()))\n\n # prepare spatial bias\n b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps))\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef model_info(model, verbose=False):\n # Plots a line-by-line description of a PyTorch model\n n_p = sum(x.numel() for x in model.parameters()) # number parameters\n n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients\n if verbose:\n print('%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma'))\n for i, (name, p) in enumerate(model.named_parameters()):\n name = name.replace('module_list.', '')\n print('%5g %40s %9s %12g %20s %10.3g %10.3g' %\n (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))\n\n try: # FLOPS\n from thop import profile\n flops = profile(deepcopy(model), inputs=(torch.zeros(1, 3, 64, 64),), verbose=False)[0] / 1E9 * 2\n fs = ', %.1f GFLOPS' % (flops * 100) # 640x640 FLOPS\n except:\n fs = ''\n\n logger.info(\n 'Model Summary: %g layers, %g parameters, %g gradients%s' % (len(list(model.parameters())), n_p, n_g, fs))\n\n\ndef load_classifier(name='resnet101', n=2):\n # Loads a pretrained model reshaped to n-class output\n model = models.__dict__[name](pretrained=True)\n\n # Display model properties\n input_size = [3, 224, 224]\n input_space = 'RGB'\n input_range = [0, 1]\n mean = [0.485, 0.456, 0.406]\n std = [0.229, 0.224, 0.225]\n for x in ['input_size', 'input_space', 'input_range', 'mean', 'std']:\n print(x + ' =', eval(x))\n\n # Reshape output to n classes\n filters = model.fc.weight.shape[1]\n model.fc.bias = nn.Parameter(torch.zeros(n), requires_grad=True)\n model.fc.weight = nn.Parameter(torch.zeros(n, filters), requires_grad=True)\n model.fc.out_features = n\n return model\n\n\ndef scale_img(img, ratio=1.0, same_shape=False): # img(16,3,256,416), r=ratio\n # scales img(bs,3,y,x) by ratio\n if ratio == 1.0:\n return img\n else:\n h, w = img.shape[2:]\n s = (int(h * ratio), int(w * ratio)) # new size\n img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize\n if not same_shape: # pad/crop img\n gs = 32 # (pixels) grid size\n h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)]\n return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean\n\n\ndef copy_attr(a, b, include=(), exclude=()):\n # Copy attributes from b to a, options to only include [...] and to exclude [...]\n for k, v in b.__dict__.items():\n if (len(include) and k not in include) or k.startswith('_') or k in exclude:\n continue\n else:\n setattr(a, k, v)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **创建数据集**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image, ExifTags\nimg_formats = ['.bmp', '.jpg', '.jpeg', '.png', '.tif', '.tiff', '.dng']\nvid_formats = ['.mov', '.avi', '.mp4', '.mpg', '.mpeg', '.m4v', '.wmv', '.mkv']\nfor orientation in ExifTags.TAGS.keys():\n if ExifTags.TAGS[orientation] == 'Orientation':\n break\n\n\ndef get_hash(files):\n # Returns a single hash value of a list of files\n return sum(os.path.getsize(f) for f in files if os.path.isfile(f))\n\n\ndef exif_size(img):\n # Returns exif-corrected PIL size\n s = img.size # (width, height)\n try:\n rotation = dict(img._getexif().items())[orientation]\n if rotation == 6: # rotation 270\n s = (s[1], s[0])\n elif rotation == 8: # rotation 90\n s = (s[1], s[0])\n except:\n pass\n\n return s\n\nclass InfiniteDataLoader(torch.utils.data.dataloader.DataLoader):\n \"\"\" Dataloader that reuses workers.\n Uses same syntax as vanilla DataLoader.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler))\n self.iterator = super().__iter__()\n\n def __len__(self):\n return len(self.batch_sampler.sampler)\n\n def __iter__(self):\n for i in range(len(self)):\n yield next(self.iterator)\n\n\nclass _RepeatSampler(object):\n \"\"\" Sampler that repeats forever.\n Args:\n sampler (Sampler)\n \"\"\"\n\n def __init__(self, sampler):\n self.sampler = sampler\n\n def __iter__(self):\n while True:\n yield from iter(self.sampler)\n\ndef create_dataloader(path, imgsz, batch_size, stride, opt, hyp=None, augment=False, cache=False, pad=0.0, rect=False,\n workers=8):\n # Make sure only the first process in DDP process the dataset first, and the following others can use the cache.\n\n dataset = LoadImagesAndLabels(path, imgsz, batch_size,\n augment=augment, # augment images\n hyp=hyp, # augmentation hyperparameters\n rect=rect, # rectangular training\n cache_images=cache,\n single_cls=False,\n stride=int(stride),\n pad=pad,\n )\n\n batch_size = min(batch_size, len(dataset))\n nw = min([os.cpu_count() // 1, batch_size if batch_size > 1 else 0, workers]) # number of workers\n\n dataloader = InfiniteDataLoader(dataset,\n batch_size=batch_size,\n num_workers=nw,\n sampler=None,\n pin_memory=True,\n collate_fn=LoadImagesAndLabels.collate_fn)\n return dataloader, dataset\n\n\nclass LoadImages: # for inference\n def __init__(self, path, img_size=640):\n p = str(Path(path)) # os-agnostic\n p = os.path.abspath(p) # absolute path\n if '*' in p:\n files = sorted(glob.glob(p)) # glob\n elif os.path.isdir(p):\n files = sorted(glob.glob(os.path.join(p, '*.*'))) # dir\n elif os.path.isfile(p):\n files = [p] # files\n else:\n raise Exception('ERROR: %s does not exist' % p)\n\n images = [x for x in files if os.path.splitext(x)[-1].lower() in img_formats]\n videos = [x for x in files if os.path.splitext(x)[-1].lower() in vid_formats]\n ni, nv = len(images), len(videos)\n\n self.img_size = img_size\n self.files = images + videos\n self.nf = ni + nv # number of files\n self.video_flag = [False] * ni + [True] * nv\n self.mode = 'images'\n if any(videos):\n self.new_video(videos[0]) # new video\n else:\n self.cap = None\n assert self.nf > 0, 'No images or videos found in %s. Supported formats are:\\nimages: %s\\nvideos: %s' % \\\n (p, img_formats, vid_formats)\n\n def __iter__(self):\n self.count = 0\n return self\n\n def __next__(self):\n if self.count == self.nf:\n raise StopIteration\n path = self.files[self.count]\n\n if self.video_flag[self.count]:\n # Read video\n self.mode = 'video'\n ret_val, img0 = self.cap.read()\n if not ret_val:\n self.count += 1\n self.cap.release()\n if self.count == self.nf: # last video\n raise StopIteration\n else:\n path = self.files[self.count]\n self.new_video(path)\n ret_val, img0 = self.cap.read()\n\n self.frame += 1\n print('video %g/%g (%g/%g) %s: ' % (self.count + 1, self.nf, self.frame, self.nframes, path), end='')\n\n else:\n # Read image\n self.count += 1\n img0 = cv2.imread(path) # BGR\n assert img0 is not None, 'Image Not Found ' + path\n print('image %g/%g %s: ' % (self.count, self.nf, path), end='')\n\n # Padded resize\n img = letterbox(img0, new_shape=self.img_size)[0]\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n # cv2.imwrite(path + '.letterbox.jpg', 255 * img.transpose((1, 2, 0))[:, :, ::-1]) # save letterbox image\n return path, img, img0, self.cap\n\n def new_video(self, path):\n self.frame = 0\n self.cap = cv2.VideoCapture(path)\n self.nframes = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT))\n\n def __len__(self):\n return self.nf # number of files\n\n\nclass LoadWebcam: # for inference\n def __init__(self, pipe=0, img_size=640):\n self.img_size = img_size\n\n if pipe == '0':\n pipe = 0 # local camera\n # pipe = 'rtsp://192.168.1.64/1' # IP camera\n # pipe = 'rtsp://username:password@192.168.1.64/1' # IP camera with login\n # pipe = 'rtsp://170.93.143.139/rtplive/470011e600ef003a004ee33696235daa' # IP traffic camera\n # pipe = 'http://wmccpinetop.axiscam.net/mjpg/video.mjpg' # IP golf camera\n\n # https://answers.opencv.org/question/215996/changing-gstreamer-pipeline-to-opencv-in-pythonsolved/\n # pipe = '\"rtspsrc location=\"rtsp://username:password@192.168.1.64/1\" latency=10 ! appsink' # GStreamer\n\n # https://answers.opencv.org/question/200787/video-acceleration-gstremer-pipeline-in-videocapture/\n # https://stackoverflow.com/questions/54095699/install-gstreamer-support-for-opencv-python-package # install help\n # pipe = \"rtspsrc location=rtsp://root:root@192.168.0.91:554/axis-media/media.amp?videocodec=h264&resolution=3840x2160 protocols=GST_RTSP_LOWER_TRANS_TCP ! rtph264depay ! queue ! vaapih264dec ! videoconvert ! appsink\" # GStreamer\n\n self.pipe = pipe\n self.cap = cv2.VideoCapture(pipe) # video capture object\n self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) # set buffer size\n\n def __iter__(self):\n self.count = -1\n return self\n\n def __next__(self):\n self.count += 1\n if cv2.waitKey(1) == ord('q'): # q to quit\n self.cap.release()\n cv2.destroyAllWindows()\n raise StopIteration\n\n # Read frame\n if self.pipe == 0: # local camera\n ret_val, img0 = self.cap.read()\n img0 = cv2.flip(img0, 1) # flip left-right\n else: # IP camera\n n = 0\n while True:\n n += 1\n self.cap.grab()\n if n % 30 == 0: # skip frames\n ret_val, img0 = self.cap.retrieve()\n if ret_val:\n break\n\n # Print\n assert ret_val, 'Camera Error %s' % self.pipe\n img_path = 'webcam.jpg'\n print('webcam %g: ' % self.count, end='')\n\n # Padded resize\n img = letterbox(img0, new_shape=self.img_size)[0]\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n return img_path, img, img0, None\n\n def __len__(self):\n return 0\n\n\nclass LoadStreams: # multiple IP or RTSP cameras\n def __init__(self, sources='streams.txt', img_size=640):\n self.mode = 'images'\n self.img_size = img_size\n\n if os.path.isfile(sources):\n with open(sources, 'r') as f:\n sources = [x.strip() for x in f.read().splitlines() if len(x.strip())]\n else:\n sources = [sources]\n\n n = len(sources)\n self.imgs = [None] * n\n self.sources = sources\n for i, s in enumerate(sources):\n # Start the thread to read frames from the video stream\n print('%g/%g: %s... ' % (i + 1, n, s), end='')\n cap = cv2.VideoCapture(eval(s) if s.isnumeric() else s)\n assert cap.isOpened(), 'Failed to open %s' % s\n w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n fps = cap.get(cv2.CAP_PROP_FPS) % 100\n _, self.imgs[i] = cap.read() # guarantee first frame\n thread = Thread(target=self.update, args=([i, cap]), daemon=True)\n print(' success (%gx%g at %.2f FPS).' % (w, h, fps))\n thread.start()\n print('') # newline\n\n # check for common shapes\n s = np.stack([letterbox(x, new_shape=self.img_size)[0].shape for x in self.imgs], 0) # inference shapes\n self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal\n if not self.rect:\n print('WARNING: Different stream shapes detected. For optimal performance supply similarly-shaped streams.')\n\n def update(self, index, cap):\n # Read next stream frame in a daemon thread\n n = 0\n while cap.isOpened():\n n += 1\n # _, self.imgs[index] = cap.read()\n cap.grab()\n if n == 4: # read every 4th frame\n _, self.imgs[index] = cap.retrieve()\n n = 0\n time.sleep(0.01) # wait time\n\n def __iter__(self):\n self.count = -1\n return self\n\n def __next__(self):\n self.count += 1\n img0 = self.imgs.copy()\n if cv2.waitKey(1) == ord('q'): # q to quit\n cv2.destroyAllWindows()\n raise StopIteration\n\n # Letterbox\n img = [letterbox(x, new_shape=self.img_size, auto=self.rect)[0] for x in img0]\n\n # Stack\n img = np.stack(img, 0)\n\n # Convert\n img = img[:, :, :, ::-1].transpose(0, 3, 1, 2) # BGR to RGB, to bsx3x416x416\n img = np.ascontiguousarray(img)\n\n return self.sources, img, img0, None\n\n def __len__(self):\n return 0 # 1E12 frames = 32 streams at 30 FPS for 30 years\n\n\nclass LoadImagesAndLabels(Dataset): # for training/testing\n def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False,\n cache_images=False, single_cls=False, stride=32, pad=0.0, rank=-1):\n try:\n f = [] # image files\n for p in path if isinstance(path, list) else [path]:\n p = str(Path(p)) # os-agnostic\n parent = str(Path(p).parent) + os.sep\n if os.path.isfile(p): # file\n with open(p, 'r') as t:\n t = t.read().splitlines()\n f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path\n elif os.path.isdir(p): # folder\n f += glob.iglob(p + os.sep + '*.*')\n else:\n raise Exception('%s does not exist' % p)\n self.img_files = sorted(\n [x.replace('/', os.sep) for x in f if os.path.splitext(x)[-1].lower() in img_formats])\n except Exception as e:\n raise Exception('Error loading data from %s: %s\\n' % (path, e))\n\n n = len(self.img_files)\n assert n > 0, 'No images found in %s' % (path)\n bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index\n nb = bi[-1] + 1 # number of batches\n\n self.n = n # number of images\n self.batch = bi # batch index of image\n self.img_size = img_size\n self.augment = augment\n self.hyp = hyp\n self.image_weights = image_weights\n self.rect = False if image_weights else rect\n self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)\n self.mosaic_border = [-img_size // 2, -img_size // 2]\n self.stride = stride\n\n # Define labels\n self.label_files = [x.replace('images', 'labels').replace(os.path.splitext(x)[-1], '.txt') for x in\n self.img_files]\n\n # Check cache\n cache_path = str(Path(self.label_files[0]).parent) + '.cache' # cached labels\n if os.path.isfile(cache_path):\n cache = torch.load(cache_path) # load\n if cache['hash'] != get_hash(self.label_files + self.img_files): # dataset changed\n cache = self.cache_labels(cache_path) # re-cache\n else:\n cache = self.cache_labels(cache_path) # cache\n\n # Get labels\n labels, shapes = zip(*[cache[x] for x in self.img_files])\n self.shapes = np.array(shapes, dtype=np.float64)\n self.labels = list(labels)\n\n # Rectangular Training https://github.com/ultralytics/yolov3/issues/232\n if self.rect:\n # Sort by aspect ratio\n s = self.shapes # wh\n ar = s[:, 1] / s[:, 0] # aspect ratio\n irect = ar.argsort()\n self.img_files = [self.img_files[i] for i in irect]\n self.label_files = [self.label_files[i] for i in irect]\n self.labels = [self.labels[i] for i in irect]\n self.shapes = s[irect] # wh\n ar = ar[irect]\n\n # Set training image shapes\n shapes = [[1, 1]] * nb\n for i in range(nb):\n ari = ar[bi == i]\n mini, maxi = ari.min(), ari.max()\n if maxi < 1:\n shapes[i] = [maxi, 1]\n elif mini > 1:\n shapes[i] = [1, 1 / mini]\n\n self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride\n\n # Cache labels\n create_datasubset, extract_bounding_boxes, labels_loaded = False, False, False\n nm, nf, ne, ns, nd = 0, 0, 0, 0, 0 # number missing, found, empty, datasubset, duplicate\n pbar = enumerate(self.label_files)\n if rank in [-1, 0]:\n pbar = tqdm(pbar)\n for i, file in pbar:\n l = self.labels[i] # label\n if l is not None and l.shape[0]:\n assert l.shape[1] == 5, '> 5 label columns: %s' % file\n assert (l >= 0).all(), 'negative labels: %s' % file\n assert (l[:, 1:] <= 1).all(), 'non-normalized or out of bounds coordinate labels: %s' % file\n if np.unique(l, axis=0).shape[0] < l.shape[0]: # duplicate rows\n nd += 1 # print('WARNING: duplicate rows in %s' % self.label_files[i]) # duplicate rows\n if single_cls:\n l[:, 0] = 0 # force dataset into single-class mode\n self.labels[i] = l\n nf += 1 # file found\n\n # Create subdataset (a smaller dataset)\n if create_datasubset and ns < 1E4:\n if ns == 0:\n create_folder(path='./datasubset')\n os.makedirs('./datasubset/images')\n exclude_classes = 43\n if exclude_classes not in l[:, 0]:\n ns += 1\n # shutil.copy(src=self.img_files[i], dst='./datasubset/images/') # copy image\n with open('./datasubset/images.txt', 'a') as f:\n f.write(self.img_files[i] + '\\n')\n\n # Extract object detection boxes for a second stage classifier\n if extract_bounding_boxes:\n p = Path(self.img_files[i])\n img = cv2.imread(str(p))\n h, w = img.shape[:2]\n for j, x in enumerate(l):\n f = '%s%sclassifier%s%g_%g_%s' % (p.parent.parent, os.sep, os.sep, x[0], j, p.name)\n if not os.path.exists(Path(f).parent):\n os.makedirs(Path(f).parent) # make new output folder\n\n b = x[1:] * [w, h, w, h] # box\n b[2:] = b[2:].max() # rectangle to square\n b[2:] = b[2:] * 1.3 + 30 # pad\n b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int)\n\n b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image\n b[[1, 3]] = np.clip(b[[1, 3]], 0, h)\n assert cv2.imwrite(f, img[b[1]:b[3], b[0]:b[2]]), 'Failure extracting classifier boxes'\n else:\n ne += 1 # print('empty labels for image %s' % self.img_files[i]) # file empty\n # os.system(\"rm '%s' '%s'\" % (self.img_files[i], self.label_files[i])) # remove\n\n if rank in [-1, 0]:\n pbar.desc = 'Scanning labels %s (%g found, %g missing, %g empty, %g duplicate, for %g images)' % (\n cache_path, nf, nm, ne, nd, n)\n if nf == 0:\n s = 'WARNING: No labels found in %s' % (os.path.dirname(file) + os.sep)\n print(s)\n assert not augment, '%s. Can not train without labels.' % s\n\n # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM)\n self.imgs = [None] * n\n if cache_images:\n gb = 0 # Gigabytes of cached images\n pbar = tqdm(range(len(self.img_files)), desc='Caching images')\n self.img_hw0, self.img_hw = [None] * n, [None] * n\n for i in pbar: # max 10k images\n self.imgs[i], self.img_hw0[i], self.img_hw[i] = load_image(self, i) # img, hw_original, hw_resized\n gb += self.imgs[i].nbytes\n pbar.desc = 'Caching images (%.1fGB)' % (gb / 1E9)\n\n def cache_labels(self, path='labels.cache'):\n # Cache dataset labels, check images and read shapes\n x = {} # dict\n pbar = tqdm(zip(self.img_files, self.label_files), desc='Scanning images', total=len(self.img_files))\n for (img, label) in pbar:\n try:\n l = []\n image = Image.open(img)\n image.verify() # PIL verify\n # _ = io.imread(img) # skimage verify (from skimage import io)\n shape = exif_size(image) # image size\n assert (shape[0] > 9) & (shape[1] > 9), 'image size <10 pixels'\n if os.path.isfile(label):\n with open(label, 'r') as f:\n l = np.array([x.split() for x in f.read().splitlines()], dtype=np.float32) # labels\n if len(l) == 0:\n l = np.zeros((0, 5), dtype=np.float32)\n x[img] = [l, shape]\n except Exception as e:\n x[img] = [None, None]\n print('WARNING: %s: %s' % (img, e))\n\n x['hash'] = get_hash(self.label_files + self.img_files)\n torch.save(x, path) # save for next time\n return x\n\n def __len__(self):\n return len(self.img_files)\n\n # def __iter__(self):\n # self.count = -1\n # print('ran dataset iter')\n # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)\n # return self\n\n def __getitem__(self, index):\n if self.image_weights:\n index = self.indices[index]\n\n hyp = self.hyp\n \n if self.mosaic:\n # Load mosaic\n img, labels = load_mosaic(self, index)\n shapes = None\n\n # MixUp https://arxiv.org/pdf/1710.09412.pdf\n if random.random() < hyp['mixup']:\n img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1))\n r = np.random.beta(8.0, 8.0) # mixup ratio, alpha=beta=8.0\n img = (img * r + img2 * (1 - r)).astype(np.uint8)\n labels = np.concatenate((labels, labels2), 0)\n\n else:\n # Load image\n img, (h0, w0), (h, w) = load_image(self, index)\n\n # Letterbox\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\n\n # Load labels\n labels = []\n x = self.labels[index]\n if x.size > 0:\n # Normalized xywh to pixel xyxy format\n labels = x.copy()\n labels[:, 1] = ratio[0] * w * (x[:, 1] - x[:, 3] / 2) + pad[0] # pad width\n labels[:, 2] = ratio[1] * h * (x[:, 2] - x[:, 4] / 2) + pad[1] # pad height\n labels[:, 3] = ratio[0] * w * (x[:, 1] + x[:, 3] / 2) + pad[0]\n labels[:, 4] = ratio[1] * h * (x[:, 2] + x[:, 4] / 2) + pad[1]\n\n if self.augment:\n # Augment imagespace\n if not self.mosaic:\n img, labels = random_perspective(img, labels,\n degrees=hyp['degrees'],\n translate=hyp['translate'],\n scale=hyp['scale'],\n shear=hyp['shear'],\n perspective=hyp['perspective'])\n\n # Augment colorspace\n augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v'])\n\n # Apply cutouts\n # if random.random() < 0.9:\n # labels = cutout(img, labels)\n\n nL = len(labels) # number of labels\n if nL:\n labels[:, 1:5] = xyxy2xywh(labels[:, 1:5]) # convert xyxy to xywh\n labels[:, [2, 4]] /= img.shape[0] # normalized height 0-1\n labels[:, [1, 3]] /= img.shape[1] # normalized width 0-1\n\n if self.augment:\n # flip up-down\n if random.random() < hyp['flipud']:\n img = np.flipud(img)\n if nL:\n labels[:, 2] = 1 - labels[:, 2]\n\n # flip left-right\n if random.random() < hyp['fliplr']:\n img = np.fliplr(img)\n if nL:\n labels[:, 1] = 1 - labels[:, 1]\n\n labels_out = torch.zeros((nL, 6))\n if nL:\n labels_out[:, 1:] = torch.from_numpy(labels)\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n return torch.from_numpy(img), labels_out, self.img_files[index], shapes\n\n @staticmethod\n def collate_fn(batch):\n img, label, path, shapes = zip(*batch) # transposed\n for i, l in enumerate(label):\n l[:, 0] = i # add target image index for build_targets()\n return torch.stack(img, 0), torch.cat(label, 0), path, shapes\n\n\n# Ancillary functions --------------------------------------------------------------------------------------------------\ndef load_image(self, index):\n # loads 1 image from dataset, returns img, original hw, resized hw\n img = self.imgs[index]\n if img is None: # not cached\n path = self.img_files[index]\n img = cv2.imread(path) # BGR\n assert img is not None, 'Image Not Found ' + path\n h0, w0 = img.shape[:2] # orig hw\n r = self.img_size / max(h0, w0) # resize image to img_size\n if r != 1: # always resize down, only resize up if training with augmentation\n interp = cv2.INTER_AREA if r < 1 and not self.augment else cv2.INTER_LINEAR\n img = cv2.resize(img, (int(w0 * r), int(h0 * r)), interpolation=interp)\n return img, (h0, w0), img.shape[:2] # img, hw_original, hw_resized\n else:\n return self.imgs[index], self.img_hw0[index], self.img_hw[index] # img, hw_original, hw_resized\n\n\ndef augment_hsv(img, hgain=0.5, sgain=0.5, vgain=0.5):\n r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains\n hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))\n dtype = img.dtype # uint8\n\n x = np.arange(0, 256, dtype=np.int16)\n lut_hue = ((x * r[0]) % 180).astype(dtype)\n lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)\n lut_val = np.clip(x * r[2], 0, 255).astype(dtype)\n\n img_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))).astype(dtype)\n cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed\n\n # Histogram equalization\n # if random.random() < 0.2:\n # for i in range(3):\n # img[:, :, i] = cv2.equalizeHist(img[:, :, i])\n\n\ndef load_mosaic(self, index):\n # loads images in a mosaic\n\n labels4 = []\n s = self.img_size\n yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y\n indices = [index] + [random.randint(0, len(self.labels) - 1) for _ in range(3)] # 3 additional image indices\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = load_image(self, index)\n\n # place img in img4\n if i == 0: # top left\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\n elif i == 1: # top right\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n elif i == 2: # bottom left\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n elif i == 3: # bottom right\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n padw = x1a - x1b\n padh = y1a - y1b\n\n # Labels\n x = self.labels[index]\n labels = x.copy()\n if x.size > 0: # Normalized xywh to pixel xyxy format\n labels[:, 1] = w * (x[:, 1] - x[:, 3] / 2) + padw\n labels[:, 2] = h * (x[:, 2] - x[:, 4] / 2) + padh\n labels[:, 3] = w * (x[:, 1] + x[:, 3] / 2) + padw\n labels[:, 4] = h * (x[:, 2] + x[:, 4] / 2) + padh\n labels4.append(labels)\n\n # Concat/clip labels\n if len(labels4):\n labels4 = np.concatenate(labels4, 0)\n # np.clip(labels4[:, 1:] - s / 2, 0, s, out=labels4[:, 1:]) # use with center crop\n np.clip(labels4[:, 1:], 0, 2 * s, out=labels4[:, 1:]) # use with random_affine\n\n # Replicate\n # img4, labels4 = replicate(img4, labels4)\n\n # Augment\n # img4 = img4[s // 2: int(s * 1.5), s // 2:int(s * 1.5)] # center crop (WARNING, requires box pruning)\n img4, labels4 = random_perspective(img4, labels4,\n degrees=self.hyp['degrees'],\n translate=self.hyp['translate'],\n scale=self.hyp['scale'],\n shear=self.hyp['shear'],\n perspective=self.hyp['perspective'],\n border=self.mosaic_border) # border to remove\n\n return img4, labels4\n\n\ndef replicate(img, labels):\n # Replicate labels\n h, w = img.shape[:2]\n boxes = labels[:, 1:].astype(int)\n x1, y1, x2, y2 = boxes.T\n s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels)\n for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices\n x1b, y1b, x2b, y2b = boxes[i]\n bh, bw = y2b - y1b, x2b - x1b\n yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y\n x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh]\n img[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0)\n\n return img, labels\n\n\ndef letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True):\n # Resize image to a 32-pixel-multiple rectangle https://github.com/ultralytics/yolov3/issues/232\n shape = img.shape[:2] # current shape [height, width]\n if isinstance(new_shape, int):\n new_shape = (new_shape, new_shape)\n\n # Scale ratio (new / old)\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n if not scaleup: # only scale down, do not scale up (for better test mAP)\n r = min(r, 1.0)\n\n # Compute padding\n ratio = r, r # width, height ratios\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\n if auto: # minimum rectangle\n dw, dh = np.mod(dw, 64), np.mod(dh, 64) # wh padding\n elif scaleFill: # stretch\n dw, dh = 0.0, 0.0\n new_unpad = (new_shape[1], new_shape[0])\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\n\n dw /= 2 # divide padding into 2 sides\n dh /= 2\n\n if shape[::-1] != new_unpad: # resize\n img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\n return img, ratio, (dw, dh)\n\n\ndef random_perspective(img, targets=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0, border=(0, 0)):\n # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10))\n # targets = [cls, xyxy]\n\n height = img.shape[0] + border[0] * 2 # shape(h,w,c)\n width = img.shape[1] + border[1] * 2\n\n # Center\n C = np.eye(3)\n C[0, 2] = -img.shape[1] / 2 # x translation (pixels)\n C[1, 2] = -img.shape[0] / 2 # y translation (pixels)\n\n # Perspective\n P = np.eye(3)\n P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y)\n P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x)\n\n # Rotation and Scale\n R = np.eye(3)\n a = random.uniform(-degrees, degrees)\n # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations\n s = random.uniform(1 - scale, 1 + scale)\n # s = 2 ** random.uniform(-scale, scale)\n R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n\n # Shear\n S = np.eye(3)\n S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)\n S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)\n\n # Translation\n T = np.eye(3)\n T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels)\n T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels)\n\n # Combined rotation matrix\n M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT\n if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed\n if perspective:\n img = cv2.warpPerspective(img, M, dsize=(width, height), borderValue=(114, 114, 114))\n else: # affine\n img = cv2.warpAffine(img, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n\n # Visualize\n # import matplotlib.pyplot as plt\n # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\n # ax[0].imshow(img[:, :, ::-1]) # base\n # ax[1].imshow(img2[:, :, ::-1]) # warped\n\n # Transform label coordinates\n n = len(targets)\n if n:\n # warp points\n xy = np.ones((n * 4, 3))\n xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1\n xy = xy @ M.T # transform\n if perspective:\n xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) # rescale\n else: # affine\n xy = xy[:, :2].reshape(n, 8)\n\n # create new boxes\n x = xy[:, [0, 2, 4, 6]]\n y = xy[:, [1, 3, 5, 7]]\n xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\n\n # # apply angle-based reduction of bounding boxes\n # radians = a * math.pi / 180\n # reduction = max(abs(math.sin(radians)), abs(math.cos(radians))) ** 0.5\n # x = (xy[:, 2] + xy[:, 0]) / 2\n # y = (xy[:, 3] + xy[:, 1]) / 2\n # w = (xy[:, 2] - xy[:, 0]) * reduction\n # h = (xy[:, 3] - xy[:, 1]) * reduction\n # xy = np.concatenate((x - w / 2, y - h / 2, x + w / 2, y + h / 2)).reshape(4, n).T\n\n # clip boxes\n xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)\n xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)\n\n # filter candidates\n i = box_candidates(box1=targets[:, 1:5].T * s, box2=xy.T)\n targets = targets[i]\n targets[:, 1:5] = xy[i]\n\n return img, targets\n\n\ndef box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.1): # box1(4,n), box2(4,n)\n # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio\n w1, h1 = box1[2] - box1[0], box1[3] - box1[1]\n w2, h2 = box2[2] - box2[0], box2[3] - box2[1]\n ar = np.maximum(w2 / (h2 + 1e-16), h2 / (w2 + 1e-16)) # aspect ratio\n return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + 1e-16) > area_thr) & (ar < ar_thr) # candidates\n\n\ndef cutout(image, labels):\n # Applies image cutout augmentation https://arxiv.org/abs/1708.04552\n h, w = image.shape[:2]\n\n def bbox_ioa(box1, box2):\n # Returns the intersection over box2 area given box1, box2. box1 is 4, box2 is nx4. boxes are x1y1x2y2\n box2 = box2.transpose()\n\n # Get the coordinates of bounding boxes\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\n\n # Intersection area\n inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \\\n (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0)\n\n # box2 area\n box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + 1e-16\n\n # Intersection over box2 area\n return inter_area / box2_area\n\n # create random masks\n scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction\n for s in scales:\n mask_h = random.randint(1, int(h * s))\n mask_w = random.randint(1, int(w * s))\n\n # box\n xmin = max(0, random.randint(0, w) - mask_w // 2)\n ymin = max(0, random.randint(0, h) - mask_h // 2)\n xmax = min(w, xmin + mask_w)\n ymax = min(h, ymin + mask_h)\n\n # apply random color mask\n image[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)]\n\n # return unobscured labels\n if len(labels) and s > 0.03:\n box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32)\n ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area\n labels = labels[ioa < 0.60] # remove >60% obscured labels\n\n return labels\n\n\ndef reduce_img_size(path='path/images', img_size=1024): # from utils.datasets import *; reduce_img_size()\n # creates a new ./images_reduced folder with reduced size images of maximum size img_size\n path_new = path + '_reduced' # reduced images path\n create_folder(path_new)\n for f in tqdm(glob.glob('%s/*.*' % path)):\n try:\n img = cv2.imread(f)\n h, w = img.shape[:2]\n r = img_size / max(h, w) # size ratio\n if r < 1.0:\n img = cv2.resize(img, (int(w * r), int(h * r)), interpolation=cv2.INTER_AREA) # _LINEAR fastest\n fnew = f.replace(path, path_new) # .replace(Path(f).suffix, '.jpg')\n cv2.imwrite(fnew, img)\n except:\n print('WARNING: image failure %s' % f)\n\n\ndef recursive_dataset2bmp(dataset='path/dataset_bmp'): # from utils.datasets import *; recursive_dataset2bmp()\n # Converts dataset to bmp (for faster training)\n formats = [x.lower() for x in img_formats] + [x.upper() for x in img_formats]\n for a, b, files in os.walk(dataset):\n for file in tqdm(files, desc=a):\n p = a + '/' + file\n s = Path(file).suffix\n if s == '.txt': # replace text\n with open(p, 'r') as f:\n lines = f.read()\n for f in formats:\n lines = lines.replace(f, '.bmp')\n with open(p, 'w') as f:\n f.write(lines)\n elif s in formats: # replace image\n cv2.imwrite(p.replace(s, '.bmp'), cv2.imread(p))\n if s != '.bmp':\n os.system(\"rm '%s'\" % p)\n\n\ndef imagelist2folder(path='path/images.txt'): # from utils.datasets import *; imagelist2folder()\n # Copies all the images in a text file (list of images) into a folder\n create_folder(path[:-4])\n with open(path, 'r') as f:\n for line in f.read().splitlines():\n os.system('cp \"%s\" %s' % (line, path[:-4]))\n print(line)\n\n\ndef create_folder(path='./new'):\n # Create folder\n if os.path.exists(path):\n shutil.rmtree(path) # delete output folder\n os.makedirs(path)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **模型建立**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nclass ModelEMA:\n \"\"\" Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models\n Keep a moving average of everything in the model state_dict (parameters and buffers).\n This is intended to allow functionality like\n https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n A smoothed version of the weights is necessary for some training schemes to perform well.\n This class is sensitive where it is initialized in the sequence of model init,\n GPU assignment and distributed training wrappers.\n \"\"\"\n\n def __init__(self, model, decay=0.9999, updates=0):\n # Create EMA\n self.ema = deepcopy(model.module if is_parallel(model) else model).eval() # FP32 EMA\n # if next(model.parameters()).device.type != 'cpu':\n # self.ema.half() # FP16 EMA\n self.updates = updates # number of EMA updates\n self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) # decay exponential ramp (to help early epochs)\n for p in self.ema.parameters():\n p.requires_grad_(False)\n\n def update(self, model):\n # Update EMA parameters\n with torch.no_grad():\n self.updates += 1\n d = self.decay(self.updates)\n\n msd = model.module.state_dict() if is_parallel(model) else model.state_dict() # model state_dict\n for k, v in self.ema.state_dict().items():\n if v.dtype.is_floating_point:\n v *= d\n v += (1. - d) * msd[k].detach()\n\n def update_attr(self, model, include=(), exclude=('process_group', 'reducer')):\n # Update EMA attributes\n copy_attr(self.ema, model, include, exclude)\n\n\ndef autopad(k, p=None): # kernel, padding\n # Pad to 'same'\n if p is None:\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\n return p\n\n\ndef DWConv(c1, c2, k=1, s=1, act=True):\n # Depthwise convolution\n return Conv(c1, c2, k, s, g=math.gcd(c1, c2), act=act)\n\n\nclass Conv(nn.Module):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Conv, self).__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.Hardswish() if act else nn.Identity()\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def fuseforward(self, x):\n return self.act(self.conv(x))\n\n\nclass Bottleneck(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(Bottleneck, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c2, 3, 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass BottleneckCSP(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSP, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.LeakyReLU(0.1, inplace=True)\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass SPP(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=(5, 9, 13)):\n super(SPP, self).__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n\n def forward(self, x):\n x = self.cv1(x)\n return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))\n\n\nclass Focus(nn.Module):\n # Focus wh information into c-space\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Focus, self).__init__()\n self.conv = Conv(c1 * 4, c2, k, s, p, g, act)\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))\n\n\nclass Concat(nn.Module):\n # Concatenate a list of tensors along dimension\n def __init__(self, dimension=1):\n super(Concat, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\nclass Flatten(nn.Module):\n # Use after nn.AdaptiveAvgPool2d(1) to remove last 2 dimensions\n @staticmethod\n def forward(x):\n return x.view(x.size(0), -1)\n\n\nclass Classify(nn.Module):\n # Classification head, i.e. x(b,c1,20,20) to x(b,c2)\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups\n super(Classify, self).__init__()\n self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1)\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False) # to x(b,c2,1,1)\n self.flat = Flatten()\n\n def forward(self, x):\n z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list\n return self.flat(self.conv(z)) # flatten to x(b,c2)\n\n\nclass CrossConv(nn.Module):\n # Cross Convolution Downsample\n def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False):\n # ch_in, ch_out, kernel, stride, groups, expansion, shortcut\n super(CrossConv, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, (1, k), (1, s))\n self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass C3(nn.Module):\n # Cross Convolution CSP\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(C3, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.LeakyReLU(0.1, inplace=True)\n self.m = nn.Sequential(*[CrossConv(c_, c_, 3, 1, g, 1.0, shortcut) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass Sum(nn.Module):\n # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, n, weight=False): # n: number of inputs\n super(Sum, self).__init__()\n self.weight = weight # apply weights boolean\n self.iter = range(n - 1) # iter object\n if weight:\n self.w = nn.Parameter(-torch.arange(1., n) / 2, requires_grad=True) # layer weights\n\n def forward(self, x):\n y = x[0] # no weight\n if self.weight:\n w = torch.sigmoid(self.w) * 2\n for i in self.iter:\n y = y + x[i + 1] * w[i]\n else:\n for i in self.iter:\n y = y + x[i + 1]\n return y\n\n\nclass GhostConv(nn.Module):\n # Ghost Convolution https://github.com/huawei-noah/ghostnet\n def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups\n super(GhostConv, self).__init__()\n c_ = c2 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, k, s, g, act)\n self.cv2 = Conv(c_, c_, 5, 1, c_, act)\n\n def forward(self, x):\n y = self.cv1(x)\n return torch.cat([y, self.cv2(y)], 1)\n\n\nclass GhostBottleneck(nn.Module):\n # Ghost Bottleneck https://github.com/huawei-noah/ghostnet\n def __init__(self, c1, c2, k, s):\n super(GhostBottleneck, self).__init__()\n c_ = c2 // 2\n self.conv = nn.Sequential(GhostConv(c1, c_, 1, 1), # pw\n DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw\n GhostConv(c_, c2, 1, 1, act=False)) # pw-linear\n self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False),\n Conv(c1, c2, 1, 1, act=False)) if s == 2 else nn.Identity()\n\n def forward(self, x):\n return self.conv(x) + self.shortcut(x)\n\n\nclass MixConv2d(nn.Module):\n # Mixed Depthwise Conv https://arxiv.org/abs/1907.09595\n def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True):\n super(MixConv2d, self).__init__()\n groups = len(k)\n if equal_ch: # equal c_ per group\n i = torch.linspace(0, groups - 1E-6, c2).floor() # c2 indices\n c_ = [(i == g).sum() for g in range(groups)] # intermediate channels\n else: # equal weight.numel() per group\n b = [c2] + [0] * groups\n a = np.eye(groups + 1, groups, k=-1)\n a -= np.roll(a, 1, axis=1)\n a *= np.array(k) ** 2\n a[0] = 1\n c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b\n\n self.m = nn.ModuleList([nn.Conv2d(c1, int(c_[g]), k[g], s, k[g] // 2, bias=False) for g in range(groups)])\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.LeakyReLU(0.1, inplace=True)\n\n def forward(self, x):\n return x + self.act(self.bn(torch.cat([m(x) for m in self.m], 1)))\n\n\nclass Ensemble(nn.ModuleList):\n # Ensemble of models\n def __init__(self):\n super(Ensemble, self).__init__()\n\n def forward(self, x, augment=False):\n y = []\n for module in self:\n y.append(module(x, augment)[0])\n # y = torch.stack(y).max(0)[0] # max ensemble\n # y = torch.cat(y, 1) # nms ensemble\n y = torch.stack(y).mean(0) # mean ensemble\n return y, None # inference, train output\n\nlogger = logging.getLogger(__name__)\n\n'''yolo主要模型'''\n\nclass Detect(nn.Module):\n stride = None # strides computed during build\n export = False # onnx export\n\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\n super(Detect, self).__init__()\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\n self.register_buffer('anchors', a) # shape(nl,na,2)\n self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv\n\n def forward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](x[i]) # conv\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n\n y = x[i].sigmoid()\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i].to(x[i].device)) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\n z.append(y.view(bs, -1, self.no))\n\n return x if self.training else (torch.cat(z, 1), x)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n\nclass Model(nn.Module):\n def __init__(self, cfg='yolov5s.yaml', ch=3, nc=None): # model, input channels, number of classes\n super(Model, self).__init__()\n if isinstance(cfg, dict):\n self.yaml = cfg # model dict\n else: # is *.yaml\n import yaml # for torch hub\n self.yaml_file = Path(cfg).name\n with open(cfg) as f:\n self.yaml = yaml.load(f, Loader=yaml.FullLoader) # model dict\n\n # Define model\n if nc and nc != self.yaml['nc']:\n print('Overriding %s nc=%g with nc=%g' % (cfg, self.yaml['nc'], nc))\n self.yaml['nc'] = nc # override yaml value\n self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist, ch_out\n # print([x.shape for x in self.forward(torch.zeros(1, ch, 64, 64))])\n\n # Build strides, anchors\n m = self.model[-1] # Detect()\n if isinstance(m, Detect):\n s = 128 # 2x min stride\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\n m.anchors /= m.stride.view(-1, 1, 1)\n check_anchor_order(m)\n self.stride = m.stride\n self._initialize_biases() # only run once\n # print('Strides: %s' % m.stride.tolist())\n\n # Init weights, biases\n initialize_weights(self)\n self.info()\n print('')\n\n def forward(self, x, augment=False, profile=False):\n if augment:\n img_size = x.shape[-2:] # height, width\n s = [1, 0.83, 0.67] # scales\n f = [None, 3, None] # flips (2-ud, 3-lr)\n y = [] # outputs\n for si, fi in zip(s, f):\n xi = scale_img(x.flip(fi) if fi else x, si)\n yi = self.forward_once(xi)[0] # forward\n # cv2.imwrite('img%g.jpg' % s, 255 * xi[0].numpy().transpose((1, 2, 0))[:, :, ::-1]) # save\n yi[..., :4] /= si # de-scale\n if fi == 2:\n yi[..., 1] = img_size[0] - yi[..., 1] # de-flip ud\n elif fi == 3:\n yi[..., 0] = img_size[1] - yi[..., 0] # de-flip lr\n y.append(yi)\n return torch.cat(y, 1), None # augmented inference, train\n else:\n return self.forward_once(x, profile) # single-scale inference, train\n\n def forward_once(self, x, profile=False):\n y, dt = [], [] # outputs\n for m in self.model:\n if m.f != -1: # if not from previous layer\n x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers\n\n if profile:\n try:\n import thop\n o = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # FLOPS\n except:\n o = 0\n t = time_synchronized()\n for _ in range(10):\n _ = m(x)\n dt.append((time_synchronized() - t) * 100)\n print('%10.1f%10.0f%10.1fms %-40s' % (o, m.np, dt[-1], m.type))\n\n x = m(x) # run\n y.append(x if m.i in self.save else None) # save output\n\n if profile:\n print('%.1fms total' % sum(dt))\n return x\n\n def _initialize_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n m = self.model[-1] # Detect() module\n for mi, s in zip(m.m, m.stride): # from\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\n b[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\n b[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n def _print_biases(self):\n m = self.model[-1] # Detect() module\n for mi in m.m: # from\n b = mi.bias.detach().view(m.na, -1).T # conv.bias(255) to (3,85)\n print(('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean()))\n\n # def _print_weights(self):\n # for m in self.model.modules():\n # if type(m) is Bottleneck:\n # print('%10.3g' % (m.w.detach().sigmoid() * 2)) # shortcut weights\n\n def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers\n print('Fusing layers... ')\n for m in self.model.modules():\n if type(m) is Conv:\n m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatability\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, 'bn') # remove batchnorm\n m.forward = m.fuseforward # update forward\n self.info()\n return self\n \n def add_nms(self): # fuse model Conv2d() + BatchNorm2d() layers\n if type(self.model[-1]) is not NMS: # if missing NMS\n print('Adding NMS module... ')\n m = NMS() # module\n m.f = -1 # from\n m.i = self.model[-1].i + 1 # index\n self.model.add_module(name='%s' % m.i, module=m) # add\n return self\n\n def info(self, verbose=False): # print model information\n model_info(self, verbose)\n\n'''根据yaml模型构建模型'''\ndef parse_model(d, ch): # model_dict, input_channels(3)\n logger.info('\\n%3s%18s%3s%10s %-40s%-30s' % ('', 'from', 'n', 'params', 'module', 'arguments'))\n anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\n na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\n\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\n for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args\n '''m就是每个模块，args就是模块的参数，n就是模块的深度，也就是模块堆叠n次'''\n '''而f代表了该模块的输入是第f个模块的输出，-1代表上一个'''\n\n m = eval(m) if isinstance(m, str) else m # eval strings\n for j, a in enumerate(args):\n try:\n '''这里可以知道第j个模块的输出'''\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\n except:\n pass\n\n n = max(round(n * gd), 1) if n > 1 else n # depth gain\n if m in [nn.Conv2d, Conv, Bottleneck, SPP, DWConv, MixConv2d, Focus, CrossConv, BottleneckCSP, C3]:\n c1, c2 = ch[f], args[0]\n\n # Normal\n # if i > 0 and args[0] != no: # channel expansion factor\n # ex = 1.75 # exponential (default 2.0)\n # e = math.log(c2 / ch[1]) / math.log(2)\n # c2 = int(ch[1] * ex ** e)\n # if m != Focus:\n\n c2 = make_divisible(c2 * gw, 8) if c2 != no else c2\n\n # Experimental\n # if i > 0 and args[0] != no: # channel expansion factor\n # ex = 1 + gw # exponential (default 2.0)\n # ch1 = 32 # ch[1]\n # e = math.log(c2 / ch1) / math.log(2) # level 1-n\n # c2 = int(ch1 * ex ** e)\n # if m != Focus:\n # c2 = make_divisible(c2, 8) if c2 != no else c2\n\n args = [c1, c2, *args[1:]]\n if m in [BottleneckCSP, C3]:\n args.insert(2, n)\n n = 1\n elif m is nn.BatchNorm2d:\n args = [ch[f]]\n elif m is Concat:\n c2 = sum([ch[-1 if x == -1 else x + 1] for x in f])\n elif m is Detect:\n args.append([ch[x + 1] for x in f])\n if isinstance(args[1], int): # number of anchors\n args[1] = [list(range(args[1] * 2))] * len(f)\n else:\n c2 = ch[f]\n\n m_ = nn.Sequential(*[m(*args) for _ in range(n)]) if n > 1 else m(*args) # module\n t = str(m)[8:-2].replace('__main__.', '') # module type\n np = sum([x.numel() for x in m_.parameters()]) # number params\n m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params\n logger.info('%3s%18s%3s%10.0f %-40s%-30s' % (i, f, n, np, t, args)) # print\n save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist\n layers.append(m_)\n ch.append(c2)\n return nn.Sequential(*layers), sorted(save)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **test**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport json\ndef test(data,weights=None,batch_size=16,imgsz=640,conf_thres=0.001,iou_thres=0.6,save_json=False,augment=False,verbose=False,\n model=None,dataloader=None,save_dir='',merge=False,save_txt=False):\n\n training=model is not None\n if training: # called by train.py\n device = next(model.parameters()).device\n\n half = device.type != 'cpu' # half precision only supported on CUDA\n if half:\n model.half()\n\n model.eval()\n with open(data) as f:\n data = yaml.load(f, Loader=yaml.FullLoader) # model dict\n check_dataset(data)\n nc=int(data['nc'])\n iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for mAP@0.5:0.95\n niou = iouv.numel()\n\n seen = 0\n names = model.names if hasattr(model, 'names') else model.module.names\n coco91class = coco80_to_coco91_class()\n s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95')\n p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.\n loss = torch.zeros(3, device=device)\n jdict, stats, ap, ap_class = [], [], [], []\n for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):\n img = img.to(device, non_blocking=True)\n img = img.half() if half else img.float() # uint8 to fp16/32\n img /= 255.0 # 0 - 255 to 0.0 - 1.0\n targets = targets.to(device)\n nb, _, height, width = img.shape # batch size, channels, height, width\n whwh = torch.Tensor([width, height, width, height]).to(device)\n\n # Disable gradients\n with torch.no_grad():\n # Run model\n t = time_synchronized()\n inf_out, train_out = model(img, augment=augment) # inference and training outputs\n t0 += time_synchronized() - t\n\n # Compute loss\n if training: # if model has loss hyperparameters\n loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # GIoU, obj, cls\n\n # Run NMS\n t = time_synchronized()\n output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge)\n t1 += time_synchronized() - t\n\n for si, pred in enumerate(output):\n labels = targets[targets[:, 0] == si, 1:]\n nl = len(labels)\n tcls = labels[:, 0].tolist() if nl else [] # target class\n seen += 1\n\n if pred is None:\n if nl:\n stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))\n continue\n\n # Append to text file\n # label format\n # Clip boxes to image bounds\n clip_coords(pred, (height, width))\n\n # Append to pycocotools JSON dictionary\n if save_json:\n # [{\"image_id\": 42, \"category_id\": 18, \"bbox\": [258.15, 41.29, 348.26, 243.78], \"score\": 0.236}, ...\n image_id = Path(paths[si]).stem\n box = pred[:, :4].clone() # xyxy\n scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape\n box = xyxy2xywh(box) # xywh\n box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner\n for p, b in zip(pred.tolist(), box.tolist()):\n jdict.append({'image_id': int(image_id) if image_id.isnumeric() else image_id,\n 'category_id': coco91class[int(p[5])],\n 'bbox': [round(x, 3) for x in b],\n 'score': round(p[4], 5)})\n\n\n\n\n\n correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)\n if nl:\n detected = [] # target indices\n tcls_tensor = labels[:, 0]\n\n # target boxes\n tbox = xywh2xyxy(labels[:, 1:5]) * whwh\n\n # Per target class\n for cls in torch.unique(tcls_tensor):\n ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices\n pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices\n\n # Search for detections\n if pi.shape[0]:\n # Prediction to target ious\n ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices\n\n # Append detections\n detected_set = set()\n for j in (ious > iouv[0]).nonzero(as_tuple=False):\n d = ti[i[j]] # detected target\n if d.item() not in detected_set:\n detected_set.add(d.item())\n detected.append(d)\n correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn\n if len(detected) == nl: # all targets already located in image\n break\n\n # Append statistics (correct, conf, pcls, tcls)\n stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))\n if batch_i < 1:\n f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename\n plot_images(img, targets, paths, str(f), names) # ground truth\n f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i)\n plot_images(img, output_to_target(output, width, height), paths, str(f), names) # predictions\n stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy\n if len(stats) and stats[0].any():\n p, r, ap, f1, ap_class = ap_per_class(*stats)\n p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1) # [P, R, AP@0.5, AP@0.5:0.95]\n mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()\n nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class\n else:\n nt = torch.zeros(1)\n\n pf = '%20s' + '%12.3g' * 6 # print format\n print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))\n\n # Print results per class\n if verbose and nc > 1 and len(stats):\n for i, c in enumerate(ap_class):\n print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))\n\n # Print speeds\n t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple\n if not training:\n print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)\n\n if save_json and len(jdict):\n f = 'detections_val2017_%s_results.json' % \\\n (weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename\n print('\\nCOCO mAP with pycocotools... saving %s...' % f)\n with open(f, 'w') as file:\n json.dump(jdict, file)\n\n try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb\n from pycocotools.coco import COCO\n from pycocotools.cocoeval import COCOeval\n\n imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files]\n cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api\n cocoDt = cocoGt.loadRes(f) # initialize COCO pred api\n cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')\n cocoEval.params.imgIds = imgIds # image IDs to evaluate\n cocoEval.evaluate()\n cocoEval.accumulate()\n cocoEval.summarize()\n map, map50 = cocoEval.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5)\n except Exception as e:\n print('ERROR: pycocotools unable to run: %s' % e)\n\n model.float() # for training\n maps = np.zeros(nc) + map\n for i, c in enumerate(ap_class):\n maps[c] = ap[i]\n return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **train**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(hyp,config,device):\n results_file='results.txt'\n last='weights_last.pt'\n best='weights_best.pt'\n os.makedirs('log')\n log_dir='log'\n cuda = device.type != 'cpu'\n epochs,batch_size,weights=config['epochs'],config['batch_size'],config['weights']\n with open(config['data']) as f:\n data_dict=yaml.load(f,Loader=yaml.FullLoader)\n\n train_path=data_dict['train']\n test_path=data_dict['val']\n nc,names=(int(data_dict['nc']),data_dict['names'])\n assert len(names) == nc\n print(weights)\n pretrained=True\n\n '''加载权重'''\n if pretrained:\n ckpt=torch.load(weights,map_location=device)\n model=Model(config['cfg'],ch=3,nc=nc).to(device)\n exclude=['anchor'] if config['cfg'] else []\n state_dict=ckpt['model']\n state_dict=intersect_dicts(state_dict,model.state_dict(),exclude=exclude)\n model.load_state_dict(state_dict,strict=False)\n logger.info('Transferred %g/%g items from %s' % (len(state_dict), len(model.state_dict()), weights)) # report\n else:\n model = Model(config['cfg'], ch=3, nc=nc).to(device)\n\n '''冻结部分参数权重'''\n freeze=['',]\n if any(freeze):\n for k, v in model.named_parameters():\n if any(x in k for x in freeze):\n print('freezing %s' % k)\n v.requires_grad = False\n\n '''调整权重衰减'''\n nbs=64\n accumulate=max(round(nbs/batch_size),1)\n hyp['weight_decay']*=batch_size*accumulate/nbs\n\n '''将模型参数分组优化'''\n pg0, pg1, pg2 = [], [], []\n for k, v in model.named_parameters():\n v.requires_grad = True\n if '.bias' in k:\n pg2.append(v) # biases\n elif '.weight' in k and '.bn' not in k:\n pg1.append(v) # apply weight decay\n else:\n pg0.append(v)\n\n '''建立优化器'''\n if config['adam']:\n optimizer = optim.Adam(pg0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum\n# elif config['Rangerlars']:\n# optimizer=RangerLars(pg0,lr=hyp['lr0'])\n else:\n optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)\n optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay\n optimizer.add_param_group({'params': pg2})\n print('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0)))\n del pg0, pg1, pg2\n\n lf = lambda x: ((1 + math.cos(x * math.pi / epochs)) / 2) * (1 - hyp['lrf']) + hyp['lrf'] # cosine\n scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)\n# if config['RangerLars']:\n# scheduler=flat_and_anneal(epochs,anneal_start=0.6,base_lr=hyp['lr0'])\n\n\n start_epoch, best_fitness = 0, 0.0\n if pretrained:\n # Optimizer\n if ckpt['optimizer'] is not None:\n optimizer.load_state_dict(ckpt['optimizer'])\n best_fitness = ckpt['best_fitness']\n\n if ckpt['training_results'] is not None:\n with open(results_file, 'w') as file:\n file.write(ckpt['training_results'])\n\n start_epoch = ckpt['epoch'] + 1\n if epochs < start_epoch:\n logger.info('%s has been trained for %g epochs. Fine-tuning for %g additional epochs.' %\n (weights, ckpt['epoch'], epochs))\n epochs += ckpt['epoch']\n del ckpt, state_dict\n\n\n gs=int(max(model.stride))\n imgsz, imgsz_test = [check_img_size(x, gs) for x in config['img_size']]\n\n ema=ModelEMA(model)\n\n dataloader,dataset=create_dataloader(train_path,imgsz,batch_size,gs,\n config,hyp=hyp,augment=True,cache=config['cache_images'],\n rect=config['rect'],workers=config['workers'])\n\n mlc=np.concatenate(dataset.labels,0)[:,0].max()\n nb = len(dataloader) # number of batches\n assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (mlc, nc, opt.data, nc - 1)\n ema.updates = start_epoch * nb // accumulate # set EMA updates\n testloader = create_dataloader(test_path, imgsz_test,batch_size, gs,config,\n hyp=hyp, augment=False, cache=config['cache_images'], rect=True,\n workers=config['workers'])[0]\n print('checkping_anchor')\n \n check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)\n\n hyp['cls'] *= nc / 80. # scale coco-tuned hyp['cls'] to current dataset\n model.nc = nc # attach number of classes to model\n model.hyp = hyp # attach hyperparameters to model\n model.gr = 1.0 # giou loss ratio (obj_loss = 1.0 or giou)\n model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) # attach class weights\n model.names = names\n\n t0 = time.time()\n nw = max(round(2 * nb), 1e3) # number of warmup iterations, max(3 epochs, 1k iterations)\n # nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training\n maps = np.zeros(nc) # mAP per class\n results = (0, 0, 0, 0, 0, 0, 0) # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification'\n \n scheduler.last_epoch = start_epoch - 1 # do not move\n scaler = amp.GradScaler(enabled=cuda)\n print('Image sizes %g train, %g test' % (imgsz, imgsz_test))\n print('Using %g dataloader workers' % dataloader.num_workers)\n print('Starting training for %g epochs...' % epochs)\n\n '''开始训练'''\n for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------\n model.train()\n\n # Update image weights (optional)\n if dataset.image_weights:\n # Generate indices\n w = model.class_weights.cpu().numpy() * (1 - maps) ** 2 # class weights\n image_weights = labels_to_image_weights(dataset.labels, nc=nc, class_weights=w)\n dataset.indices = random.choices(range(dataset.n), weights=image_weights,\n k=dataset.n)\n mloss = torch.zeros(4, device=device)\n pbar = enumerate(dataloader)\n print(('\\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets', 'img_size'))\n pbar = tqdm(pbar, total=nb)\n\n optimizer.zero_grad()\n\n for i, (imgs, targets, paths, _) in pbar: # batch -------------------------------------------------------------\n ni = i + nb * epoch # number integrated batches (since train start)\n imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0\n\n # Warmup\n if ni <= nw:\n xi = [0, nw] # x interp\n # model.gr = np.interp(ni, xi, [0.0, 1.0]) # giou loss ratio (obj_loss = 1.0 or giou)\n accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round())\n for j, x in enumerate(optimizer.param_groups):\n # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0\n x['lr'] = np.interp(ni, xi, [0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])\n if 'momentum' in x:\n x['momentum'] = np.interp(ni, xi, [0.9, hyp['momentum']])\n\n if config['multi_scale']:\n sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size\n sf = sz / max(imgs.shape[2:]) # scale factor\n if sf != 1:\n ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple)\n imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)\n\n '''进行amp'''\n with amp.autocast(enabled=cuda):\n pred = model(imgs) # forward\n loss, loss_items = compute_loss(pred, targets.to(device), model) # loss scaled by batch_size\n\n scaler.scale(loss).backward()\n if ni % accumulate == 0:\n scaler.step(optimizer) # optimizer.step\n scaler.update()\n optimizer.zero_grad()\n if ema:\n ema.update(model)\n\n mloss = (mloss * i + loss_items) / (i + 1) # update mean losses\n mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB)\n s = ('%10s' * 2 + '%10.4g' * 6) % (\n '%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1])\n pbar.set_description(s)\n\n if ni < 3:\n f = str(log_dir / ('train_batch%g.jpg' % ni)) # filename\n result = plot_images(images=imgs, targets=targets, paths=paths, fname=f)\n\n lr = [x['lr'] for x in optimizer.param_groups] # for tensorboard\n \n scheduler.step()\n if ema:\n ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'gr', 'names', 'stride'])\n final_epoch = epoch + 1 == epochs\n if final_epoch: # Calculate mAP\n [os.remove(x) for x in glob.glob(str(log_dir / 'test_batch*_pred.jpg')) if os.path.exists(x)]\n results, maps, times = test(config['data'],\n batch_size=batch_size,\n imgsz=imgsz_test,\n model=ema.ema,\n dataloader=testloader,\n save_dir=log_dir)\n with open(results_file, 'a') as f:\n f.write(s + '%10.4g' * 7 % results + '\\n')\n\n fi = fitness(np.array(results).reshape(1, -1)) # fitness_i = weighted combination of [P, R, mAP, F1]\n if fi > best_fitness:\n best_fitness = fi\n\n with open(results_file, 'r') as f: # create checkpoint\n ckpt = {'epoch': epoch,\n 'best_fitness': best_fitness,\n 'training_results': f.read(),\n 'model': ema.ema,\n 'optimizer': None if final_epoch else optimizer.state_dict()}\n\n torch.save(ckpt, last)\n if best_fitness == fi:\n torch.save(ckpt, best)\n del ckpt\n\n plot_results(save_dir=log_dir) # save as results.png\n print('%g epochs completed in %.3f hours.\\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600))\n\n torch.cuda.empty_cache()\n return results\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\ndef attempt_load(weights, map_location=None):\n # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a\n model = Ensemble()\n for w in weights if isinstance(weights, list) else [weights]:\n model.append(torch.load(w, map_location=map_location)['model'].float().fuse().eval()) # load FP32 model\n\n if len(model) == 1:\n return model[-1] # return model\n else:\n print('Ensemble created with %s\\n' % weights)\n for k in ['names', 'stride']:\n setattr(model, k, getattr(model[-1], k))\n return model # return ensemble\nos.makedirs('output')\ndetect_config={\n 'weights':'yolov5s.pt',\n 'source':'inference_images',\n 'output':'output',\n 'img-size':640,\n 'conf-thres':0.4,\n 'iou_thres':0.5,\n 'view-img':True,\n 'save-txt':True,\n 'augment':True}\n'''weights是个列表的话，则相当于进行一个model embedding'''\n'''augment为True的话，相当于进行一个TTA'''\ndef detect(detect_config):\n out,source,weights,view_img,save_txt,imgsz=config['output'],config['source'],config['weights'],config\n device=torch.device('cuda:0')\n half=device.type!='cpu'\n model=attempt_load(weights,map_location=device)\n imgsz=check_img_size(imgsz,s=model.stride.max())\n if half:\n model.half()\n classify = False\n if classify:\n modelc = load_classifier(name='resnet101', n=2) # initialize\n modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']) # load weights\n modelc.to(device).eval()\n\n # Set Dataloader\n vid_path, vid_writer = None, None\n\n save_img = True\n dataset = LoadImages(source, img_size=imgsz)\n names = model.module.names if hasattr(model, 'module') else model.names\n colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(names))]\n\n # Run inference\n t0 = time.time()\n img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img\n _ = model(img.half() if half else img) if device.type != 'cpu' else None\n for path, img, im0s, vid_cap in dataset:\n img = torch.from_numpy(img).to(device)\n img = img.half() if half else img.float() # uint8 to fp16/32\n img /= 255.0 # 0 - 255 to 0.0 - 1.0\n if img.ndimension() == 3:\n img = img.unsqueeze(0)\n\n # Inference\n t1 = time_synchronized()\n pred = model(img, augment=detect_config['augment'])[0]\n\n # Apply NMS\n pred = non_max_suppression(pred, detect_config['conf_thres'], detect_config['iou_thres'])\n t2 = time_synchronized()\n\n # Apply Classifier\n if classify:\n pred = apply_classifier(pred, modelc, img, im0s)\n\n # Process detections\n for i, det in enumerate(pred): # detections per image\n\n p, s, im0 = path, '', im0s\n\n save_path = str(Path(out) / Path(p).name)\n txt_path = str(Path(out) / Path(p).stem) + ('_%g' % dataset.frame if dataset.mode == 'video' else '')\n s += '%gx%g ' % img.shape[2:] # print string\n gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh\n if det is not None and len(det):\n # Rescale boxes from img_size to im0 size\n det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round()\n\n # Print results\n for c in det[:, -1].unique():\n n = (det[:, -1] == c).sum() # detections per class\n s += '%g %ss, ' % (n, names[int(c)]) # add to string\n\n # Write results\n for *xyxy, conf, cls in reversed(det):\n if save_txt: # Write to file\n xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh\n with open(txt_path + '.txt', 'a') as f:\n f.write(('%g ' * 5 + '\\n') % (cls, *xywh)) # label format\n\n if save_img or view_img: # Add bbox to image\n label = '%s %.2f' % (names[int(cls)], conf)\n plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=3)\n\n # Print time (inference + NMS)\n print('%sDone. (%.3fs)' % (s, t2 - t1))\n\n # Stream results\n if view_img:\n cv2.imshow(p, im0)\n if cv2.waitKey(1) == ord('q'): # q to quit\n raise StopIteration\n\n # Save results (image with detections)\n if save_img:\n if dataset.mode == 'images':\n cv2.imwrite(save_path, im0)\n else:\n if vid_path != save_path: # new video\n vid_path = save_path\n if isinstance(vid_writer, cv2.VideoWriter):\n vid_writer.release() # release previous video writer\n\n fourcc = 'mp4v' # output video codec\n fps = vid_cap.get(cv2.CAP_PROP_FPS)\n w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*fourcc), fps, (w, h))\n vid_writer.write(im0)\n\n\n\n print('Done. (%.3fs)' % (time.time() - t0))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from shutil import copyfile\ntrain_path='/kaggle/input/global-wheat-detection/train'\n# os.makedirs('images')\n# os.makedirs('images/train')\n\n# os.makedirs('labels')\n# os.makedirs('labels/train')\n\n# names=get_image_files(train_path)\n# train_df=pd.read_csv('/kaggle/input/global-wheat-detection/train.csv')\n# from collections import defaultdict\n\n# image_bbox = defaultdict(list)\n# for i in range(train_df.shape[0]):\n# info = train_df.iloc[i]\n# origin_bbox = info['bbox']\n# bbox = origin_bbox[1:-1].split(',')\n# try:\n# box = [float(bbox[0].strip()), float(bbox[1].strip()), float(bbox[2].strip()), float(bbox[3].strip())]\n# except ValueError:\n# print(origin_bbox)\n# print(bbox)\n\n# image_id = info['image_id']\n# image_bbox[image_id].append(box)\n\n# def convert(size,bbox):\n# dh=1./size[0]\n# dw=1./size[1]\n# xc=int(bbox[0]+bbox[2]/2)\n# yc=int(bbox[1]+bbox[3]/2)\n# w=int(bbox[2])\n# h=int(bbox[3])\n\n# xc=xc*dw\n# w=w*dw\n# yc=yc*dh\n# h=h*dh\n# return (xc,yc,w,h)\n\n\n# for name in names:\n# name=str(name)\n# image=cv2.imread(name)\n\n# file=name.split('/')[-1].split('.')[0]\n# obj=os.path.join('images','train',file+'.jpg')\n\n# copyfile(name,obj)\n\n# with open(os.path.join('labels','train',file+'.txt'),'w') as f:\n# bboxs=image_bbox[file]\n# for bbox in bboxs:\n# box=convert(image.shape,bbox)\n# f.write(\"0\" + \" \" + \" \".join([str(a) for a in box]) + '\\n')\n\ndf = pd.read_csv('../input/global-wheat-detection/train.csv')\nbboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n df[column] = bboxs[:,i]\ndf.drop(columns=['bbox'], inplace=True)\ndf['x_center'] = df['x'] + df['w']/2\ndf['y_center'] = df['y'] + df['h']/2\ndf['classes'] = 0\nfrom tqdm.auto import tqdm\nimport shutil as sh\ndf = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]\nindex = list(set(df.image_id))\n\nsource = 'train'\nif True:\n for fold in [0]:\n val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]\n for name,mini in tqdm(df.groupby('image_id')):\n if name in val_index:\n path2save = 'val2017/'\n else:\n path2save = 'train2017/'\n if not os.path.exists('convertor/fold{}/labels/'.format(fold)+path2save):\n os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save)\n with open('convertor/fold{}/labels/'.format(fold)+path2save+name+\".txt\", 'w+') as f:\n row = mini[['classes','x_center','y_center','w','h']].astype(float).values\n row = row/1024\n row = row.astype(str)\n for j in range(len(row)):\n text = ' '.join(row[j])\n f.write(text)\n f.write(\"\\n\")\n if not os.path.exists('convertor/fold{}/images/{}'.format(fold,path2save)):\n os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save))\n sh.copy(\"../input/global-wheat-detection/{}/{}.jpg\".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name))\ndata_yaml={\n 'nc':1,\n 'train':'./convertor/fold0/images/train2017/',\n 'val':'./convertor/fold0/images/val2017/',\n 'names':['xiaomai']\n}\n\nimport yaml\nf=open('data.yaml','w')\nyaml.dump(data_yaml,f)\nf.close()\n\nconfig={\n 'weights':'/kaggle/input/yolov5-config/init_code/yolov5x.pt',\n 'cfg':'../input/yolov5-config/init_code/yolov5x.yaml',\n 'data':'coco128.yaml',\n 'epochs':10,\n 'img_size':[1024,1024],\n 'rect':False,\n 'cache_images':False,\n 'multi-scale': False,\n 'logdir':'logging',\n 'adam':False,\n 'single-cls':False,\n 'sync-bn':False,\n 'local_rank':-1,\n 'workers':2,\n 'hyp':'/kaggle/input/yolov5-config/hyp.finetune.yaml',\n 'Rangerlars':False,\n \n}\nconfig['batch_size']=2\nconfig['world_size']=1\nconfig['global_rank']=-1\nconfig['data']='data.yaml'\nconfig['multi_scale']=False\n\nconfig['weights']='../input/new-yolo/myyolov5x.pt'\n\ndevice=torch.device('cuda:0')\nwith open(config['hyp']) as f:\n hyp=yaml.load(f,Loader=yaml.FullLoader)\n\ntrain(hyp,config,device)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('results.txt','rb') as f:\n lines=f.readlines()\n for line in lines:\n print(line)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# detect_config={\n# 'weights':'weight_best.pt',\n# 'source':'/kaggle/input/test',\n# 'output':'output',\n# 'img-size':640,\n# 'conf-thres':0.4,\n# 'iou_thres':0.5,\n# 'view-img':True,\n# 'save-txt':True,\n# 'augment':False}\n# detect(detect_config)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}