{"cells":[{"metadata":{"_uuid":"827a6908-3284-4850-a940-9e3880abea1d","_cell_guid":"7b20cdc3-c8eb-4d66-a1f0-1a466eae4876","trusted":true},"cell_type":"markdown","source":"Nothing special about this notebook just wanted to share my training pipeline for YOLOv5, I hope it may helpful for someone :]\n\n","execution_count":null},{"metadata":{"_uuid":"e498dec7-aaa7-4aae-b39b-520ed67847af","_cell_guid":"3a3358a3-d661-4e4e-8178-72a5a013d5b2","trusted":true},"cell_type":"markdown","source":"## Setup","execution_count":null},{"metadata":{"_uuid":"55f7062a-f105-464b-9c99-20fc14981645","_cell_guid":"5853a8dc-5439-4a70-8026-fb988746e819","trusted":true},"cell_type":"code","source":"! git clone https://github.com/NVIDIA/apex","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab71a0dc-f965-4142-84e4-da1a0246482b","_cell_guid":"9df12d7c-0616-4ce1-be38-3d122e303f12","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/104229\n\n!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./apex","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b0ca641-6855-4641-9a55-cc1144dc4c13","_cell_guid":"96318313-14cb-41bc-8aaf-d4b35b23e2f2","trusted":true},"cell_type":"code","source":"!git clone https://github.com/ultralytics/yolov5\n!mv yolov5/* ./\n!pip install -r requirements.txt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ee5033d-27e7-4cbd-9e35-e07a1ee95f9a","_cell_guid":"0fbc5a50-b628-4815-9b39-6f70afc1cf51","trusted":true},"cell_type":"markdown","source":"## Dataset and K-fold CV","execution_count":null},{"metadata":{"_uuid":"39ab4a16-8477-4ba1-89ef-1c245efd8f07","_cell_guid":"3232e666-f67b-4246-b056-2bdf36707135","trusted":true},"cell_type":"markdown","source":"### this section of the code orignally from https://www.kaggle.com/orkatz2/yolov5-train and I C&P it from :D https://www.kaggle.com/nvnnghia/yolov5-pseudo-labeling","execution_count":null},{"metadata":{"_uuid":"431f58c2-262b-46ac-baf8-bd81143bf78e","_cell_guid":"de8b1178-9e25-4b5d-87c8-526a26af298e","trusted":true},"cell_type":"code","source":"def convertTrainLabel(dir_train, num_of_fold, fold_number=0):\n    df = pd.read_csv(dir_train)\n    bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        df[column] = bboxs[:,i]\n    df.drop(columns=['bbox'], inplace=True)\n    df['x_center'] = df['x'] + df['w']/2\n    df['y_center'] = df['y'] + df['h']/2\n    df['classes'] = 0\n    from tqdm.auto import tqdm\n    import shutil as sh\n    df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]\n    \n    index = list(sorted(df.image_id))\n    \n    source = 'train'\n    if True:\n        for fold in [fold_number]:\n            val_index = index[len(index)*fold//num_of_fold:len(index)*(fold+1)//num_of_fold]\n            for name,mini in tqdm(df.groupby('image_id')):\n                if name in val_index:\n                    path2save = 'val/'\n                else:\n                    path2save = 'train/'\n                if not os.path.exists('convertor/fold{}/labels/'.format(fold)+path2save):\n                    os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save)\n                with open('convertor/fold{}/labels/'.format(fold)+path2save+name+\".txt\", 'w+') as f:\n                    row = mini[['classes','x_center','y_center','w','h']].astype(float).values\n                    row = row/1024\n                    row = row.astype(str)\n                    for j in range(len(row)):\n                        text = ' '.join(row[j])\n                        f.write(text)\n                        f.write(\"\\n\")\n                if not os.path.exists('convertor/fold{}/images/{}'.format(fold,path2save)):\n                    os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save))\n                sh.copy(\"../input/global-wheat-detection/{}/{}.jpg\".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b8af14a-811f-4ce9-9e5f-667131e8c5e1","_cell_guid":"ee326514-e904-44f4-9155-f5f653fe8a46","trusted":true},"cell_type":"markdown","source":"## Training pipeline","execution_count":null},{"metadata":{"_uuid":"d2ec8b0f-7243-4503-a2d4-f1d1c446fd28","_cell_guid":"d52f7dc5-df79-404f-8eb4-bf86a3a5decc","trusted":true},"cell_type":"markdown","source":"This section of the code from orginal YOLOv5 git repo: https://github.com/ultralytics/yolov5","execution_count":null},{"metadata":{"_uuid":"bd1d010c-6839-4e9e-b705-fd643280390d","_cell_guid":"a95986fd-a930-4edd-884f-0f437880f63a","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport torch.distributed as dist\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as lr_scheduler\nimport torch.utils.data\nfrom torch.utils.tensorboard import SummaryWriter\n#import test  # import test.py to get mAP after each epoch\nfrom models.yolo import Model\nfrom utils import google_utils\nfrom utils.datasets import *\nfrom utils.utils import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7504264d-05a7-4953-8700-aee220821b27","_cell_guid":"4aaf31d6-491b-493e-8cff-ba55892e652b","trusted":true},"cell_type":"code","source":"def test(data,\n         weights=None,\n         batch_size=16,\n         imgsz=640,\n         conf_thres=0.001,\n         iou_thres=0.6,  # for NMS\n         save_json=False,\n         single_cls=False,\n         augment=False,\n         verbose=False,\n         model=None,\n         dataloader=None,\n         merge=False):\n    # Initialize/load model and set device\n    if model is None:\n        training = False\n        device = torch_utils.select_device(opt.device, batch_size=batch_size)\n\n        # Remove previous\n        for f in glob.glob('test_batch*.jpg'):\n            os.remove(f)\n\n        # Load model\n        google_utils.attempt_download(weights)\n        model = torch.load(weights, map_location=device)['model'].float()  # load to FP32\n        torch_utils.model_info(model)\n        model.fuse()\n        model.to(device)\n\n        # Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99\n        # if device.type != 'cpu' and torch.cuda.device_count() > 1:\n        #     model = nn.DataParallel(model)\n\n    else:  # called by train.py\n        training = True\n        device = next(model.parameters()).device  # get model device\n\n    # Half\n    half = device.type != 'cpu' and torch.cuda.device_count() == 1  # half precision only supported on single-GPU\n    if half:\n        model.half()  # to FP16\n\n    # Configure\n    model.eval()\n    with open(data) as f:\n        data = yaml.load(f, Loader=yaml.FullLoader)  # model dict\n    nc = 1 if single_cls else int(data['nc'])  # number of classes\n    iouv = torch.linspace(0.5, 0.95, 10).to(device)  # iou vector for mAP@0.5:0.95\n    niou = iouv.numel()\n\n    # Dataloader\n    if dataloader is None:  # not training\n        merge = opt.merge  # use Merge NMS\n        img = torch.zeros((1, 3, imgsz, imgsz), device=device)  # init img\n        _ = model(img.half() if half else img) if device.type != 'cpu' else None  # run once\n        path = data['test'] if opt.task == 'test' else data['val']  # path to val/test images\n        dataloader = create_dataloader(path, imgsz, batch_size, int(max(model.stride)), opt,\n                                       hyp=None, augment=False, cache=False, pad=0.5, rect=True)[0]\n\n    seen = 0\n    names = model.names if hasattr(model, 'names') else model.module.names\n    coco91class = coco80_to_coco91_class()\n    s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95')\n    p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.\n    loss = torch.zeros(3, device=device)\n    jdict, stats, ap, ap_class = [], [], [], []\n    for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):\n        img = img.to(device)\n        img = img.half() if half else img.float()  # uint8 to fp16/32\n        img /= 255.0  # 0 - 255 to 0.0 - 1.0\n        targets = targets.to(device)\n        nb, _, height, width = img.shape  # batch size, channels, height, width\n        whwh = torch.Tensor([width, height, width, height]).to(device)\n\n        # Disable gradients\n        with torch.no_grad():\n            # Run model\n            t = torch_utils.time_synchronized()\n            inf_out, train_out = model(img, augment=augment)  # inference and training outputs\n            t0 += torch_utils.time_synchronized() - t\n\n            # Compute loss\n            if training:  # if model has loss hyperparameters\n                loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3]  # GIoU, obj, cls\n\n            # Run NMS\n            t = torch_utils.time_synchronized()\n            output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, merge=merge)\n            t1 += torch_utils.time_synchronized() - t\n\n        # Statistics per image\n        for si, pred in enumerate(output):\n            labels = targets[targets[:, 0] == si, 1:]\n            nl = len(labels)\n            tcls = labels[:, 0].tolist() if nl else []  # target class\n            seen += 1\n\n            if pred is None:\n                if nl:\n                    stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))\n                continue\n\n            # Append to text file\n            # with open('test.txt', 'a') as file:\n            #    [file.write('%11.5g' * 7 % tuple(x) + '\\n') for x in pred]\n\n            # Clip boxes to image bounds\n            clip_coords(pred, (height, width))\n\n            # Append to pycocotools JSON dictionary\n            if save_json:\n                # [{\"image_id\": 42, \"category_id\": 18, \"bbox\": [258.15, 41.29, 348.26, 243.78], \"score\": 0.236}, ...\n                image_id = int(Path(paths[si]).stem.split('_')[-1])\n                box = pred[:, :4].clone()  # xyxy\n                scale_coords(img[si].shape[1:], box, shapes[si][0], shapes[si][1])  # to original shape\n                box = xyxy2xywh(box)  # xywh\n                box[:, :2] -= box[:, 2:] / 2  # xy center to top-left corner\n                for p, b in zip(pred.tolist(), box.tolist()):\n                    jdict.append({'image_id': image_id,\n                                  'category_id': coco91class[int(p[5])],\n                                  'bbox': [round(x, 3) for x in b],\n                                  'score': round(p[4], 5)})\n\n            # Assign all predictions as incorrect\n            correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)\n            if nl:\n                detected = []  # target indices\n                tcls_tensor = labels[:, 0]\n\n                # target boxes\n                tbox = xywh2xyxy(labels[:, 1:5]) * whwh\n\n                # Per target class\n                for cls in torch.unique(tcls_tensor):\n                    ti = (cls == tcls_tensor).nonzero().view(-1)  # prediction indices\n                    pi = (cls == pred[:, 5]).nonzero().view(-1)  # target indices\n\n                    # Search for detections\n                    if pi.shape[0]:\n                        # Prediction to target ious\n                        ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1)  # best ious, indices\n\n                        # Append detections\n                        for j in (ious > iouv[0]).nonzero():\n                            d = ti[i[j]]  # detected target\n                            if d not in detected:\n                                detected.append(d)\n                                correct[pi[j]] = ious[j] > iouv  # iou_thres is 1xn\n                                if len(detected) == nl:  # all targets already located in image\n                                    break\n\n            # Append statistics (correct, conf, pcls, tcls)\n            stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))\n\n        # Plot images\n        if batch_i < 1:\n            f = 'test_batch%g_gt.jpg' % batch_i  # filename\n            plot_images(img, targets, paths, f, names)  # ground truth\n            f = 'test_batch%g_pred.jpg' % batch_i\n            plot_images(img, output_to_target(output, width, height), paths, f, names)  # predictions\n\n    # Compute statistics\n    stats = [np.concatenate(x, 0) for x in zip(*stats)]  # to numpy\n    if len(stats):\n        p, r, ap, f1, ap_class = ap_per_class(*stats)\n        p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1)  # [P, R, AP@0.5, AP@0.5:0.95]\n        mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()\n        nt = np.bincount(stats[3].astype(np.int64), minlength=nc)  # number of targets per class\n    else:\n        nt = torch.zeros(1)\n\n    # Print results\n    pf = '%20s' + '%12.3g' * 6  # print format\n    print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))\n\n    # Print results per class\n    if verbose and nc > 1 and len(stats):\n        for i, c in enumerate(ap_class):\n            print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))\n\n    # Print speeds\n    t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size)  # tuple\n    if not training:\n        print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)\n\n    # Save JSON\n    if save_json and map50 and len(jdict):\n        imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataloader.dataset.img_files]\n        f = 'detections_val2017_%s_results.json' % \\\n            (weights.split(os.sep)[-1].replace('.pt', '') if weights else '')  # filename\n        print('\\nCOCO mAP with pycocotools... saving %s...' % f)\n        with open(f, 'w') as file:\n            json.dump(jdict, file)\n\n        try:\n            from pycocotools.coco import COCO\n            from pycocotools.cocoeval import COCOeval\n\n            # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb\n            cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0])  # initialize COCO ground truth api\n            cocoDt = cocoGt.loadRes(f)  # initialize COCO pred api\n\n            cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')\n            cocoEval.params.imgIds = imgIds  # image IDs to evaluate\n            cocoEval.evaluate()\n            cocoEval.accumulate()\n            cocoEval.summarize()\n            map, map50 = cocoEval.stats[:2]  # update results (mAP@0.5:0.95, mAP@0.5)\n        except:\n            print('WARNING: pycocotools must be installed with numpy==1.17 to run correctly. '\n                  'See https://github.com/cocodataset/cocoapi/issues/356')\n\n    # Return results\n    model.float()  # for training\n    maps = np.zeros(nc) + map\n    for i, c in enumerate(ap_class):\n        maps[c] = ap[i]\n    return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b63623f-17fe-49b2-b571-6d31df5aa464","_cell_guid":"445d2f28-77c7-465b-888d-be02b1bd5386","trusted":true},"cell_type":"code","source":"mixed_precision = True\ntry:  # Mixed precision training https://github.com/NVIDIA/apex\n    from apex import amp\nexcept:\n    print('Apex recommended for faster mixed precision training: https://github.com/NVIDIA/apex')\n    mixed_precision = False  # not installed\n\nwdir = 'weights' + os.sep  # weights dir\nos.makedirs(wdir, exist_ok=True)\nlast = wdir + 'last.pt'\nbest = wdir + 'best.pt'\nresults_file = 'results.txt'\n\n# Hyperparameters\nhyp = {'lr0': 0.01,  # initial learning rate (SGD=1E-2, Adam=1E-3)\n       'momentum': 0.937,  # SGD momentum\n       'weight_decay': 5e-4,  # optimizer weight decay\n       'giou': 0.05,  # giou loss gain\n       'cls': 0.58,  # cls loss gain\n       'cls_pw': 1.0,  # cls BCELoss positive_weight\n       'obj': 1.0,  # obj loss gain (*=img_size/320 if img_size != 320)\n       'obj_pw': 1.0,  # obj BCELoss positive_weight\n       'iou_t': 0.20,  # iou training threshold\n       'anchor_t': 4.0,  # anchor-multiple threshold\n       'fl_gamma': 0.0,  # focal loss gamma (efficientDet default is gamma=1.5)\n       'hsv_h': 0.014,  # image HSV-Hue augmentation (fraction)\n       'hsv_s': 0.68,  # image HSV-Saturation augmentation (fraction)\n       'hsv_v': 0.36,  # image HSV-Value augmentation (fraction)\n       'degrees': 0.0,  # image rotation (+/- deg)\n       'translate': 0.0,  # image translation (+/- fraction)\n       'scale': 0.5,  # image scale (+/- gain)\n       'shear': 0.0}  # image shear (+/- deg)\nprint(hyp)\n\n# Overwrite hyp with hyp*.txt (optional)\nf = glob.glob('hyp*.txt')\nif f:\n    print('Using %s' % f[0])\n    for k, v in zip(hyp.keys(), np.loadtxt(f[0])):\n        hyp[k] = v\n\n# Print focal loss if gamma > 0\nif hyp['fl_gamma']:\n    print('Using FocalLoss(gamma=%g)' % hyp['fl_gamma'])\n\n\ndef train(hyp):\n    epochs = opt.epochs  # 300\n    batch_size = opt.batch_size  # 64\n    weights = opt.weights  # initial training weights\n\n    # Configure\n    init_seeds(1)\n    with open(opt.data) as f:\n        data_dict = yaml.load(f, Loader=yaml.FullLoader)  # model dict\n    train_path = data_dict['train']\n    test_path = data_dict['val']\n    nc = 1 if opt.single_cls else int(data_dict['nc'])  # number of classes\n\n    # Remove previous results\n    for f in glob.glob('*_batch*.jpg') + glob.glob(results_file):\n        os.remove(f)\n\n    # Create model\n    model = Model(opt.cfg).to(device)\n    assert model.md['nc'] == nc, '%s nc=%g classes but %s nc=%g classes' % (opt.data, nc, opt.cfg, model.md['nc'])\n    model.names = data_dict['names']\n\n    # Image sizes\n    gs = int(max(model.stride))  # grid size (max stride)\n    imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size]  # verify imgsz are gs-multiples\n\n    # Optimizer\n    nbs = 64  # nominal batch size\n    accumulate = max(round(nbs / batch_size), 1)  # accumulate loss before optimizing\n    hyp['weight_decay'] *= batch_size * accumulate / nbs  # scale weight_decay\n    pg0, pg1, pg2 = [], [], []  # optimizer parameter groups\n    for k, v in model.named_parameters():\n        if v.requires_grad:\n            if '.bias' in k:\n                pg2.append(v)  # biases\n            elif '.weight' in k and '.bn' not in k:\n                pg1.append(v)  # apply weight decay\n            else:\n                pg0.append(v)  # all else\n\n    optimizer = optim.Adam(pg0, lr=hyp['lr0']) if opt.adam else \\\n        optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)\n    optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']})  # add pg1 with weight_decay\n    optimizer.add_param_group({'params': pg2})  # add pg2 (biases)\n    print('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0)))\n    del pg0, pg1, pg2\n\n    # Load Model\n    google_utils.attempt_download(weights)\n    start_epoch, best_fitness = 0, 0.0\n    if weights.endswith('.pt'):  # pytorch format\n        ckpt = torch.load(weights, map_location=device)  # load checkpoint\n\n        # load model\n        try:\n            ckpt['model'] = {k: v for k, v in ckpt['model'].float().state_dict().items()\n                             if model.state_dict()[k].shape == v.shape}  # to FP32, filter\n            model.load_state_dict(ckpt['model'], strict=False)\n        except KeyError as e:\n            s = \"%s is not compatible with %s. Specify --weights '' or specify a --cfg compatible with %s.\" \\\n                % (opt.weights, opt.cfg, opt.weights)\n            raise KeyError(s) from e\n\n        # load optimizer\n        if ckpt['optimizer'] is not None:\n            optimizer.load_state_dict(ckpt['optimizer'])\n            best_fitness = ckpt['best_fitness']\n\n        # load results\n        if ckpt.get('training_results') is not None:\n            with open(results_file, 'w') as file:\n                file.write(ckpt['training_results'])  # write results.txt\n\n        start_epoch = ckpt['epoch'] + 1\n        del ckpt\n\n    # Mixed precision training https://github.com/NVIDIA/apex\n    if mixed_precision:\n        model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)\n\n    # Scheduler https://arxiv.org/pdf/1812.01187.pdf\n    lf = lambda x: (((1 + math.cos(x * math.pi / epochs)) / 2) ** 1.0) * 0.9 + 0.1  # cosine\n    scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)\n    scheduler.last_epoch = start_epoch - 1  # do not move\n    # https://discuss.pytorch.org/t/a-problem-occured-when-resuming-an-optimizer/28822\n    # plot_lr_scheduler(optimizer, scheduler, epochs)\n\n    # Initialize distributed training\n    if device.type != 'cpu' and torch.cuda.device_count() > 1 and torch.distributed.is_available():\n        dist.init_process_group(backend='nccl',  # distributed backend\n                                init_method='tcp://127.0.0.1:9999',  # init method\n                                world_size=1,  # number of nodes\n                                rank=0)  # node rank\n        model = torch.nn.parallel.DistributedDataParallel(model)\n        # pip install torch==1.4.0+cu100 torchvision==0.5.0+cu100 -f https://download.pytorch.org/whl/torch_stable.html\n\n    # Trainloader\n    dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt,\n                                            hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect)\n    mlc = np.concatenate(dataset.labels, 0)[:, 0].max()  # max label class\n    assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Correct your labels or your model.' % (mlc, nc, opt.cfg)\n\n    # Testloader\n    testloader = create_dataloader(test_path, imgsz_test, batch_size, gs, opt,\n                                            hyp=hyp, augment=False, cache=opt.cache_images, rect=True)[0]\n\n    # Model parameters\n    hyp['cls'] *= nc / 80.  # scale coco-tuned hyp['cls'] to current dataset\n    model.nc = nc  # attach number of classes to model\n    model.hyp = hyp  # attach hyperparameters to model\n    model.gr = 1.0  # giou loss ratio (obj_loss = 1.0 or giou)\n    model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device)  # attach class weights\n\n    # Class frequency\n    labels = np.concatenate(dataset.labels, 0)\n    c = torch.tensor(labels[:, 0])  # classes\n    # cf = torch.bincount(c.long(), minlength=nc) + 1.\n    # model._initialize_biases(cf.to(device))\n    if tb_writer:\n        plot_labels(labels)\n        tb_writer.add_histogram('classes', c, 0)\n\n    # Check anchors\n    if not opt.noautoanchor:\n        check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)\n\n    # Exponential moving average\n    ema = torch_utils.ModelEMA(model)\n\n    # Start training\n    t0 = time.time()\n    nb = len(dataloader)  # number of batches\n    n_burn = max(3 * nb, 1e3)  # burn-in iterations, max(3 epochs, 1k iterations)\n    maps = np.zeros(nc)  # mAP per class\n    results = (0, 0, 0, 0, 0, 0, 0)  # 'P', 'R', 'mAP', 'F1', 'val GIoU', 'val Objectness', 'val Classification'\n    print('Image sizes %g train, %g test' % (imgsz, imgsz_test))\n    print('Using %g dataloader workers' % dataloader.num_workers)\n    print('Starting training for %g epochs...' % epochs)\n    # torch.autograd.set_detect_anomaly(True)\n    for epoch in range(start_epoch, epochs):  # epoch ------------------------------------------------------------------\n        model.train()\n\n        # Update image weights (optional)\n        if dataset.image_weights:\n            w = model.class_weights.cpu().numpy() * (1 - maps) ** 2  # class weights\n            image_weights = labels_to_image_weights(dataset.labels, nc=nc, class_weights=w)\n            dataset.indices = random.choices(range(dataset.n), weights=image_weights, k=dataset.n)  # rand weighted idx\n\n        mloss = torch.zeros(4, device=device)  # mean losses\n        print(('\\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'GIoU', 'obj', 'cls', 'total', 'targets', 'img_size'))\n        pbar = tqdm(enumerate(dataloader), total=nb)  # progress bar\n        for i, (imgs, targets, paths, _) in pbar:  # batch -------------------------------------------------------------\n            ni = i + nb * epoch  # number integrated batches (since train start)\n            imgs = imgs.to(device).float() / 255.0  # uint8 to float32, 0 - 255 to 0.0 - 1.0\n\n            # Burn-in\n            if ni <= n_burn:\n                xi = [0, n_burn]  # x interp\n                # model.gr = np.interp(ni, xi, [0.0, 1.0])  # giou loss ratio (obj_loss = 1.0 or giou)\n                accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round())\n                for j, x in enumerate(optimizer.param_groups):\n                    # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0\n                    x['lr'] = np.interp(ni, xi, [0.1 if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])\n                    if 'momentum' in x:\n                        x['momentum'] = np.interp(ni, xi, [0.9, hyp['momentum']])\n\n            # Multi-scale\n            if opt.multi_scale:\n                sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs  # size\n                sf = sz / max(imgs.shape[2:])  # scale factor\n                if sf != 1:\n                    ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]]  # new shape (stretched to gs-multiple)\n                    imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)\n\n            # Forward\n            pred = model(imgs)\n\n            # Loss\n            loss, loss_items = compute_loss(pred, targets.to(device), model)\n            if not torch.isfinite(loss):\n                print('WARNING: non-finite loss, ending training ', loss_items)\n                return results\n\n            # Backward\n            if mixed_precision:\n                with amp.scale_loss(loss, optimizer) as scaled_loss:\n                    scaled_loss.backward()\n            else:\n                loss.backward()\n\n            # Optimize\n            if ni % accumulate == 0:\n                optimizer.step()\n                optimizer.zero_grad()\n                ema.update(model)\n\n            # Print\n            mloss = (mloss * i + loss_items) / (i + 1)  # update mean losses\n            mem = '%.3gG' % (torch.cuda.memory_cached() / 1E9 if torch.cuda.is_available() else 0)  # (GB)\n            s = ('%10s' * 2 + '%10.4g' * 6) % (\n                '%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1])\n            pbar.set_description(s)\n\n            # Plot\n            if ni < 3:\n                f = 'train_batch%g.jpg' % ni  # filename\n                result = plot_images(images=imgs, targets=targets, paths=paths, fname=f)\n                if tb_writer and result is not None:\n                    tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch)\n                    # tb_writer.add_graph(model, imgs)  # add model to tensorboard\n\n            # end batch ------------------------------------------------------------------------------------------------\n\n        # Scheduler\n        scheduler.step()\n\n        # mAP\n        ema.update_attr(model)\n        final_epoch = epoch + 1 == epochs\n        if not opt.notest or final_epoch:  # Calculate mAP\n            #results, maps, times = test.test(opt.data,\n            #                                 batch_size=batch_size,\n            #                                 imgsz=imgsz_test,\n            #                                 save_json=final_epoch and opt.data.endswith(os.sep + 'coco.yaml'),\n            #                                 model=ema.ema,\n            #                                 single_cls=opt.single_cls,\n            #                                 dataloader=testloader)\n            results, maps, times = test(opt.data,\n                                        batch_size=batch_size,\n                                        imgsz=imgsz_test,\n                                        save_json=final_epoch and opt.data.endswith(os.sep + 'coco.yaml'),\n                                        model=ema.ema,\n                                        single_cls=opt.single_cls,\n                                        dataloader=testloader)\n\n\n        # Write\n        with open(results_file, 'a') as f:\n            f.write(s + '%10.4g' * 7 % results + '\\n')  # P, R, mAP, F1, test_losses=(GIoU, obj, cls)\n        if len(opt.name) and opt.bucket:\n            os.system('gsutil cp results.txt gs://%s/results/results%s.txt' % (opt.bucket, opt.name))\n\n        # Tensorboard\n        if tb_writer:\n            tags = ['train/giou_loss', 'train/obj_loss', 'train/cls_loss',\n                    'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/F1',\n                    'val/giou_loss', 'val/obj_loss', 'val/cls_loss']\n            for x, tag in zip(list(mloss[:-1]) + list(results), tags):\n                tb_writer.add_scalar(tag, x, epoch)\n\n        # Update best mAP\n        fi = fitness(np.array(results).reshape(1, -1))  # fitness_i = weighted combination of [P, R, mAP, F1]\n        if fi > best_fitness:\n            best_fitness = fi\n\n        # Save model\n        save = (not opt.nosave) or (final_epoch and not opt.evolve)\n        if save:\n            with open(results_file, 'r') as f:  # create checkpoint\n                ckpt = {'epoch': epoch,\n                        'best_fitness': best_fitness,\n                        'training_results': f.read(),\n                        'model': ema.ema.module if hasattr(model, 'module') else ema.ema,\n                        'optimizer': None if final_epoch else optimizer.state_dict()}\n\n            # Save last, best and delete\n            torch.save(ckpt, last)\n            if (best_fitness == fi) and not final_epoch:\n                torch.save(ckpt, best)\n            del ckpt\n\n        # end epoch ----------------------------------------------------------------------------------------------------\n    # end training\n\n    n = opt.name\n    if len(n):\n        n = '_' + n if not n.isnumeric() else n\n        fresults, flast, fbest = 'results%s.txt' % n, wdir + 'last%s.pt' % n, wdir + 'best%s.pt' % n\n        for f1, f2 in zip([wdir + 'last.pt', wdir + 'best.pt', 'results.txt'], [flast, fbest, fresults]):\n            if os.path.exists(f1):\n                os.rename(f1, f2)  # rename\n                ispt = f2.endswith('.pt')  # is *.pt\n                strip_optimizer(f2) if ispt else None  # strip optimizer\n                os.system('gsutil cp %s gs://%s/weights' % (f2, opt.bucket)) if opt.bucket and ispt else None  # upload\n\n    if not opt.evolve:\n        plot_results()  # save as results.png\n    print('%g epochs completed in %.3f hours.\\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600))\n    dist.destroy_process_group() if device.type != 'cpu' and torch.cuda.device_count() > 1 else None\n    torch.cuda.empty_cache()\n    return results","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19f7be6e-0df4-4e8d-ab05-d2526450036c","_cell_guid":"7774095a-f025-4ad6-b787-bc48443cf513","trusted":true},"cell_type":"markdown","source":"## Run training and hyperparameters\n\nIf you want to resume the trainining preiodically just use resume=True","execution_count":null},{"metadata":{"_uuid":"765b3ec7-bfda-4559-8f1c-3336a50891ea","_cell_guid":"d4efbe60-03d1-4d33-9e13-217d6e78ffaf","trusted":true},"cell_type":"code","source":"class opt:\n    train_csv = '../input/global-wheat-detection/train.csv'\n    #train_csv = '../input/train_noise/train_filtered.csv'\n    #train_csv = '../input/train_aug/train_aug.csv'\n    fold = 5\n    epochs = 1\n    batch_size = 2\n    cfg = '../input/yolov5x/yolov5x.yaml'\n    data = '../input/yolov5x/wheat0.yaml'\n    img_size = [1024] # default 640 640\n    weights = '../input/yolov5x/yolov5x.pt' # Defaul '', initial weights path, initial coco weights \n    name = 'yolov5x_fold0' # Default '', renames results.txt to results_name.txt if supplie\n    rect = False # default False, rectangular training\n    resume = False # Default False, resume training from last.pt \n    nosave = False # Default False, only save final checkpoint\n    notest = False # Default False, only test final epoch\n    noautoanchor = False # Default true, disabke autoanchor check\n    evolve = False # Default False, evolve hyperparameters\n    bucket = '' # Defaul '', gsutil bucket\n    cache_images = False # Default False, cache images for faster training\n    device = '' # Defaul '', cuda device, i.e. 0 or 0,1,2,3 or cpu\n    adam = False # Defaul False, adam optimizer\n    multi_scale = True # Default False, vary img-size +/- 50%\n    single_cls = False # Default False, train as single-class dataset","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"deb097eb-0030-4852-af9c-07d7625d4797","_cell_guid":"7c94646e-36af-4722-b1ac-41dd440a701f","trusted":true},"cell_type":"code","source":"if __name__ == '__main__': \n    #check_git_status()\n    \n    # covert train labels\n    convertTrainLabel(opt.train_csv, opt.fold, 0)\n\n    opt.weights = last if opt.resume else opt.weights\n    opt.cfg = check_file(opt.cfg)  # check file\n    opt.data = check_file(opt.data)  # check file\n    \n    opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size)))  # extend to 2 sizes (train, test)\n    device = torch_utils.select_device(opt.device, apex=mixed_precision, batch_size=opt.batch_size)\n    if device.type == 'cpu':\n        mixed_precision = False\n\n    # Train\n    if not opt.evolve:\n        tb_writer = SummaryWriter(comment=opt.name)\n        print('Start Tensorboard with \"tensorboard --logdir=runs\", view at http://localhost:6006/')\n        train(hyp)\n\n    # Evolve hyperparameters (optional)\n    else:\n        tb_writer = None\n        opt.notest, opt.nosave = True, True  # only test/save final epoch\n        if opt.bucket:\n            os.system('gsutil cp gs://%s/evolve.txt .' % opt.bucket)  # download evolve.txt if exists\n\n        for _ in range(10):  # generations to evolve\n            if os.path.exists('evolve.txt'):  # if evolve.txt exists: select best hyps and mutate\n                # Select parent(s)\n                parent = 'single'  # parent selection method: 'single' or 'weighted'\n                x = np.loadtxt('evolve.txt', ndmin=2)\n                n = min(5, len(x))  # number of previous results to consider\n                x = x[np.argsort(-fitness(x))][:n]  # top n mutations\n                w = fitness(x) - fitness(x).min()  # weights\n                if parent == 'single' or len(x) == 1:\n                    # x = x[random.randint(0, n - 1)]  # random selection\n                    x = x[random.choices(range(n), weights=w)[0]]  # weighted selection\n                elif parent == 'weighted':\n                    x = (x * w.reshape(n, 1)).sum(0) / w.sum()  # weighted combination\n\n                # Mutate\n                mp, s = 0.9, 0.2  # mutation probability, sigma\n                npr = np.random\n                npr.seed(int(time.time()))\n                g = np.array([1, 1, 1, 1, 1, 1, 1, 0, .1, 1, 0, 1, 1, 1, 1, 1, 1, 1])  # gains\n                ng = len(g)\n                v = np.ones(ng)\n                while all(v == 1):  # mutate until a change occurs (prevent duplicates)\n                    v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0)\n                for i, k in enumerate(hyp.keys()):  # plt.hist(v.ravel(), 300)\n                    hyp[k] = x[i + 7] * v[i]  # mutate\n\n            # Clip to limits\n            keys = ['lr0', 'iou_t', 'momentum', 'weight_decay', 'hsv_s', 'hsv_v', 'translate', 'scale', 'fl_gamma']\n            limits = [(1e-5, 1e-2), (0.00, 0.70), (0.60, 0.98), (0, 0.001), (0, .9), (0, .9), (0, .9), (0, .9), (0, 3)]\n            for k, v in zip(keys, limits):\n                hyp[k] = np.clip(hyp[k], v[0], v[1])\n\n            # Train mutation\n            results = train(hyp.copy())\n\n            # Write mutation results\n            print_mutation(hyp, results, opt.bucket)\n\n            # Plot results\n            # plot_evolution_results(hyp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dc41c06-624c-4b72-9789-7ee670e86496","_cell_guid":"69e9ee7d-7539-4910-b67c-79b6d6aad8ab","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}