{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Yolo Tutorial","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"CNF={'id':f'yolo0620x',\n     'train':False,\n     'checkPredcitions':True,\n     'infer':False,\n     'createPseudo':True,\n     'train_jigsaw':False,\n     'balance_sampling':False,\n     'max_epoch':100,#100\n     'fold':2,\n     'exdata0a':False,\n     'efd_level':0,\n     'save_best_ckpt_num':1,\n     'frac':1,\n     'target_fold':1, # いつもは0\n     'cutmix':False,\n     'batch_size':4,#4*3 728->4\n     'mixup_last_n':1, # 0 -> no mixup 素のbatchsizeは12の時は3が丁度良かった\n     'aws':False,\n     \n     'lastCKPT':f\"../input/large0618fold1/*last*\", # CNFにlastCKPTキーがあれば続きからトレーニング\n     'bestCKPT':f\"../input/large0618fold1/*best*\", # CNFにbestCKPTキーがあれば読み込んで性能評価、Inferとか\n#      'lastCKPT':f\"../input/trainovermergedimage5fold/*last*\", # CNFにlastCKPTキーがあれば続きからトレーニング\n#      'bestCKPT':f\"../input/trainovermergedimage5fold/*best*\", # CNFにbestCKPTキーがあれば読み込んで性能評価、Inferとか\n     'SEED':42\n     }\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"YoloのInstall","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\n!git clone https://github.com/ultralytics/yolov5  # clone repo\n!pip install -r yolov5/requirements.txt  # install dependencies\n#%cd yolov5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom ast import literal_eval\n\ntrain_df = pd.read_csv(\"../input/global-wheat-detection/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert(size, box):\n    dw = 1. / size[0];dh = 1. / size[1];x = (box[0] + box[1]) / 2.0;y = (box[2] + box[3]) / 2.0\n    w = box[1] - box[0];h = box[3] - box[2]\n    x = x * dw;w = w * dw;y = y * dh;h = h * dh\n    return [x, y, w, h]\n\ndef convert_to_yolo_label(coco_format_box, w = 1024, h = 1024):\n    bbox = literal_eval(coco_format_box)\n    xmin = bbox[0];xmax = bbox[0] + bbox[2];ymin = bbox[1];ymax = bbox[1] + bbox[3]\n    b = (float(xmin), float(xmax), float(ymin), float(ymax))\n    yolo_box = convert((w, h), b)\n    if np.max(yolo_box) > 1 or np.min(yolo_box) < 0: # Take this opportunity to check that conversion works\n        print(\"BOX HAS AN ISSUE\")\n    return yolo_box\n\ntrain_df['yolo_box'] = train_df.bbox.apply(convert_to_yolo_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\ndef append_folds(marking):\n\n    skf = StratifiedKFold(n_splits=CNF['fold'], shuffle=True, random_state=42)\n        \n    df_folds = marking[['image_id']].copy()\n    df_folds.loc[:, 'bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds.loc[:, 'stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n    df_folds.loc[:, 'fold'] = 0\n\n    for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n        df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n\n    df_folds=df_folds.reset_index()\n\n    marking=marking.merge(df_folds[['image_id','fold','stratify_group']],how='left',on='image_id')\n        \n        \n    return marking\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bboxなし画像IDリストnegative_image_ids\nfrom glob import glob\nall_imgs = glob(\"../input/global-wheat-detection/train/*.jpg\")\nall_imgs = [i.split(\"/\")[-1].replace(\".jpg\", \"\") for i in all_imgs]\npositive_imgs = train_df.image_id.unique()\nnegative_image_ids = list(set(all_imgs) - set(positive_imgs))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=append_folds(train_df) if 'fold' not in train_df.columns else train_df\n\n# subsample\nif CNF['frac']!=1:\n    image_list = pd.DataFrame(train_df.image_id.unique()).sample(frac=CNF['frac'],random_state=42).values.squeeze()\n    train_df   = train_df[train_df.image_id.isin(image_list)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ホームディレクトリにて環境設定","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\n\nif os.path.basename(os.getcwd())=='yolov5':\n    os.chdir('..')  \n    \n# working---yolov5---data---GWHDfold0.yaml\n#         |               |-GWHDfold1.yaml\n#         |\n#         |-gwhd---images---trn_fold0--many jpgs\n#                |        |-val_fold0--many jpgs\n#                |        |-trn_fold1--many jpgs...\n#                |        |-val_fold1--many jpgs...\n#                |\n#                |-labels---trn_fold0--many txt (bbox)\n#                         |-val_fold0--many txt\n\n#\nfor fold in range(CNF['fold']):\n    \n    !echo -e \"train: ../gwhd/images/trn_fold{fold}\\nval: ../gwhd/images/val_fold{fold}\\nnc: 1\\nnames: ['wheat']\" > ./yolov5/data/GWHDfold{fold}.yaml\n    !mkdir -p ./gwhd/images/trn_fold{fold} ./gwhd/images/val_fold{fold} ./gwhd/labels/trn_fold{fold} ./gwhd/labels/val_fold{fold}\n    \n    trn_images=train_df[train_df.fold!=fold].image_id.unique()\n    val_images=train_df[train_df.fold==fold].image_id.unique()\n    \n    [ shutil.copyfile(f\"../input/global-wheat-detection/train/{image_id}.jpg\",f\"./gwhd/images/trn_fold{fold}/{image_id}.jpg\") for image_id in trn_images ]\n    [ shutil.copyfile(f\"../input/global-wheat-detection/train/{image_id}.jpg\",f\"./gwhd/images/val_fold{fold}/{image_id}.jpg\") for image_id in val_images ]\n    [ shutil.copyfile(f\"../input/global-wheat-detection/train/{image_id}.jpg\",f\"./gwhd/images/trn_fold{fold}/{image_id}.jpg\") for image_id in negative_image_ids ]\n    \n    for trn_or_val,image_ids in zip(['trn','val'],[trn_images,val_images]):\n        \n        for img_id in image_ids:\n            filt_df = train_df.query(\"image_id == @img_id\") \n            all_boxes = filt_df.yolo_box.values\n            if os.path.exists(f\"./gwhd/labels/{trn_or_val}_fold{fold}/{img_id}.txt\")==False:\n                with open(f\"./gwhd/labels/{trn_or_val}_fold{fold}/{img_id}.txt\", 'a') as file: \n                    for i in all_boxes:\n                        new_line = (\"0 %s %s %s %s \\n\" % tuple(i));file.write(new_line)  \n\n    for img_id in negative_image_ids:\n\n        with open(f\"./gwhd/labels/trn_fold{fold}/{img_id}.txt\", 'w') as fp: \n            pass\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.basename(os.getcwd())!='yolov5':\n    os.chdir('yolov5')  \n    \nfor fold in range(CNF['fold'])[:1]:\n    \n    # クラス数を1に設定\n    !sed \"s/nc: 80/nc: 1/\" ../yolov5/models/yolov5s.yaml > ../yolov5/models/yolov5s_{CNF['id']}_fold{fold}.yaml\n    \n    !python train.py --img 1024 --batch 16 --epochs {CNF['max_epoch']} --data ./data/GWHDfold{fold}.yaml --cfg ./models/yolov5s_{CNF['id']}_fold{fold}.yaml --weights yolov5s.pt --name {CNF['id']}_fold{fold} --nosave --cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cp weights/last_{CNF['id']}_fold{fold}.pt ..","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cd ..\n!rm -rf ./gwhd\n!rm -rf ./yolov5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !python detect.py --weights ./weights/last_xlarge0619_fold0.pt --img 1024 --conf 0.5 --source ../gwhd/images/val_fold0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import Image\nfnames =glob(\"inference/output/*\")\nImage(fnames[50],width=512)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Albumentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose(\n        [   \n            A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),\n            A.OneOf([\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),\n                A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.9),],p=0.9),\n            A.ToGray(p=0.01),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=CNF['nnimSize'], width=CNF['nnimSize'], p=1),\n            A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),#これはCNF['nnimSize']かえても変えてない\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0, \n        bbox_params=A.BboxParams(format='pascal_voc',min_area=0,min_visibility=0,label_fields=['labels'])\n    )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=CNF['nnimSize'], width=CNF['nnimSize'], p=1.0),ToTensorV2(p=1.0),], p=1.0, \n        bbox_params=A.BboxParams(format='pascal_voc',min_area=0, min_visibility=0,label_fields=['labels']))\n\ndef get_test_transforms():\n    return A.Compose([A.Resize(height=CNF['nnimSize'], width=CNF['nnimSize'], p=1.0),ToTensorV2(p=1.0)])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, jigsaw, marking, transforms=None, test=False):\n        super().__init__()\n        \n        # check df index is 0,1,2,...\n        assert np.all( marking.index.tolist()==np.arange(len(marking)) )\n        self.jigsaw=jigsaw ; self.marking=marking ; self.transforms=transforms ; self.test=test\n        \n        self.image_ids = marking.merge_id.drop_duplicates().values if jigsaw else marking.image_id.drop_duplicates().values\n        \n        if jigsaw:\n            self.markingPerMergeId = dict(tuple(marking.groupby('merge_id')))\n        self.markingPerImageId = dict(tuple(marking.groupby('image_id')))\n        \n        self.infer = not (\"x\" in marking.columns) # test prediction mode\n        \n    def __getitem__(self, index: int):\n        \n        image_id = self.image_ids[index]\n        \n        if self.infer:\n            image=readImage(image_id)\n            return self.transforms(**{'image': image})['image'],image_id # test prediction mode\n            \n        if CNF['cutmix']:\n            if self.test or random.random() > 0.5:\n                image, boxes = self.load_image_and_boxes(index)\n            else:\n                image, boxes = self.load_cutmix_image_and_boxes(index)\n        else:\n            image, boxes = self.load_image_and_boxes(index)\n            \n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10): # トランスフォームしすぎてBBOX無くなる場合を避ける\n                sample = self.transforms(**{'image': image,'bboxes': target['boxes'],'labels': labels})\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return len(self.image_ids) \n        \n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        \n        if self.jigsaw:\n            \n            # 画像統合\n            _mgdf=merge_id2image_id_df[image_id] # ここでいう引数のimage_idはmerge_idのことね\n            _ims=[ readImage(image_id) for image_id in _mgdf.image_id]\n            if len(_ims)==6: # _mgdf.NUM_PATCH_SRCと同じ\n                image=np.vstack([np.hstack([_ims[0],_ims[1],_ims[2]]),np.hstack([_ims[3],_ims[4],_ims[5]])])\n                #start_tate, start_yoko = np.random.randint(0, 1024), np.random.randint(0, 2048)\n            elif len(_ims)==4:\n                image=np.vstack([np.hstack([_ims[0],_ims[1]]),np.hstack([_ims[2],_ims[3]])])\n                #start_tate, start_yoko = np.random.randint(0, 1024), np.random.randint(0, 1024)\n            else:\n                image=np.hstack([_ims[0],_ims[1]])\n                #start_tate, start_yoko = 0, np.random.randint(0, 1024)\n                \n            records = self.markingPerMergeId[image_id] # ここのインデックスはmerge_idの意味\n            boxes = records[['mb0','mb1','mb2','mb3']].values\n               \n            #NO_USE_ALBUM\n            if False:\n                # 1024x1024 crop\n                image=image[start_tate:(start_tate+1024),start_yoko:(start_yoko+1024),:]\n                # box crop\n                boxes[:,0]-=start_yoko;boxes[:,1]-=start_tate;boxes[:,2]-=start_yoko;boxes[:,3]-=start_tate\n                boxes=boxes.clip(0,1023)\n                boxes=boxes[ (boxes[:,2]-boxes[:,0])*(boxes[:,3]-boxes[:,1]) != 0 ]              \n            \n        else:\n            \n            image=readImage(image_id)\n            records = self.markingPerImageId[image_id]\n            boxes = records[['x', 'y', 'w', 'h']].values\n            boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n            boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n            \n        return image, boxes\n\n    def load_cutmix_image_and_boxes(self, index, imsize=1024):\n        \n        # 画面四分割して左上はindexの画像、他３つは適当にとってきた画像\n        \"\"\" \n        This implementation of cutmix author:  https://www.kaggle.com/nvnnghia \n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize\n        s = imsize // 2\n    \n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y\n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]\n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32) # まず白で埋めた画像をつくる\n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)\n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n\n            boxes[:, 0] += padw\n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n\n        result_boxes = np.concatenate(result_boxes, 0)\n        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[np.where((result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1]) > 0)]\n        return result_image, result_boxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# marking = append_folds(marking,is_jigsaw_group_split=CNF['train_jigsaw'])\n# marking = marking.dropna()\n# fold_number=0\n# train_dataset = DatasetRetriever(\n#         jigsaw=CNF['train_jigsaw'],\n#         marking=marking[marking['fold'] != fold_number].reset_index(drop=True),\n#         transforms=get_train_transforms(),\n#         test=False,\n#     )\n\n# for  cnt,(image, target, image_id) in enumerate(train_dataset):\n\n#     plt.figure(figsize=(10,10))\n#     image=image.permute(1, 2, 0).numpy()\n#     for box in target['boxes'].numpy():\n\n#         box=box.astype('int')\n#         cv2.rectangle(image, (box[1], box[0]), (box[3], box[2]), (1, 1, 1), 2) \n\n#     plt.imshow(image)\n    \n#     if cnt>100:\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def createDataSet(marking):\n\n    fold_number = CNF['target_fold']\n\n    train_dataset = DatasetRetriever(\n        jigsaw=CNF['train_jigsaw'],\n        marking=marking[marking['fold'] != fold_number].reset_index(drop=True),\n        transforms=get_train_transforms(),\n        test=False,\n    )\n\n    validation_dataset = DatasetRetriever(\n        jigsaw=False,\n        marking=marking[marking['fold'] == fold_number].reset_index(drop=True),\n        transforms=get_valid_transforms(),\n        test=True,\n    )\n    \n    return train_dataset, validation_dataset,None\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fitter","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    \n    def __init__(self, model, device, config):\n        \n        self.hist={'trn_loss':[],'val_loss':[],'tst_loss':[]}\n        \n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{config.folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n        \n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n\n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        \n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch(train_loader)\n            self.hist['trn_loss'].append(summary_loss.avg)\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            #if e==(self.config.n_epochs-1):\n            self.save(f'{self.base_dir}/last-checkpoint.bin') # 毎回保存は処理が重いかな\n\n            t = time.time()\n            summary_loss = self.validation(validation_loader)\n            self.hist['val_loss'].append(summary_loss.avg)\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-CNF['save_best_ckpt_num']]:os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n                \n        return self.hist\n\n    def validation(self, val_loader,is_test=False,return_loss=False):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    ValOrTst = '** Test **' if is_test else 'Val'\n                    print(f'{ValOrTst} Step {step}/{len(val_loader)}, ' + f'summary_loss: {summary_loss.avg:.5f}, ' + f'time: {(time.time() - t):.5f}', end='\\r')\n                    \n            with torch.no_grad():\n                \n                images = torch.stack(images)\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                boxes  = [target['boxes'].to(self.device).float()  for target in targets]\n                labels = [target['labels'].to(self.device).float() for target in targets]\n\n                loss, _, _ = self.model(images, boxes, labels)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        if return_loss:\n            return summary_loss,loss\n        else:\n            return summary_loss\n\n    def train_one_epoch(self, train_loader):\n        \n        self.scheduler.step()\n        \n        self.model.train()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(f'Train Step {step}/{len(train_loader)}, ' + f'summary_loss: {summary_loss.avg:.5f}, ' + f'time: {(time.time() - t):.5f}', end='\\r')\n            \n            images = torch.stack(images)\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n            boxes  = [target['boxes'].to(self.device).float()  for target in targets]\n            labels = [target['labels'].to(self.device).float() for target in targets]\n\n            self.optimizer.zero_grad()\n            \n            loss, _, _ = self.model(images, boxes, labels, num_mixup=CNF['mixup_last_n'])\n            \n            loss.backward()\n\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss\n    \n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.model.state_dict(),'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,}, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path, map_location=device) # GPUの時にチェックしてない\n        self.model.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n        \n    def predict(self, loader,score_threshold,isTrn=True):\n        self.model.eval();predictions=[]\n        for images, targets, image_ids in loader:\n                \n            with torch.no_grad():\n                \n                images = torch.stack(images);batch_size = images.shape[0];images = images.to(self.device).float()\n                det = self.model(images, torch.tensor([1]*images.shape[0]).float().to(device))\n\n                for i in range(images.shape[0]):\n                    boxes = det[i].detach().cpu().numpy()[:,:4];scores = det[i].detach().cpu().numpy()[:,4]\n                    indexes = np.where(scores >= score_threshold)[0]\n                    boxes = boxes[indexes];boxes[:, 2] = boxes[:, 2] + boxes[:, 0];boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                    predictions.append({ 'boxes': boxes[indexes],'scores': scores[indexes],})\n\n        return predictions\n    \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainGlobalConfig:\n \n    num_workers = 8 if CNF['aws'] else 2 \n    batch_size = CNF['batch_size']\n    n_epochs = CNF['max_epoch']# n_epochs = 40\n    lr = 1\n    folder = f'effdet-{CNF[\"id\"]}'\n    verbose = True\n    verbose_step = 1\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = False  # do scheduler.step after validation stage loss\n    \n    import functools\n    \n    def mylr(epch):\n        \n        base=0.0001\n        _lr = [10]*4+[5]*4+[1]*4\n        CYCLE_LEN=len(_lr)\n        return _lr[epch%CYCLE_LEN]*base\n    \n    SchedulerClass = torch.optim.lr_scheduler.LambdaLR\n    \n    scheduler_params = dict(lr_lambda = mylr,)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ndef run_training(net,train_dataset,validation_dataset,test_dataset,trn_sample_weight):\n    \n    net.to(device)\n    \n    trn_sampler=WeightedRandomSampler(trn_sample_weight,len(trn_sample_weight),replacement=True) if CNF['balance_sampling'] else RandomSampler(train_dataset)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        sampler=trn_sampler,\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n        collate_fn=collate_fn,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_fn,\n    )\n    \n    fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)\n    \n    if 'lastCKPT' in CNF.keys():\n        fitter.load(glob(CNF['lastCKPT'])[0]) # 続きからトレーニング\n    \n    train_history=fitter.fit(train_loader, val_loader)\n    \n    return train_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from effdet import get_efficientdet_config,   DetBenchEval, EfficientDet #,DetBenchTrain,\nfrom effdet.efficientdet import HeadNet\nfrom torch import nn\nclass GWDModel(nn.Module):\n\n    def __init__(self,isTrn=False):\n        super(GWDModel, self).__init__()\n        \n        self.config = get_efficientdet_config(f'tf_efficientdet_d{CNF[\"efd_level\"]}')\n        self.net = EfficientDet(self.config, pretrained_backbone=False)\n        checkpoint = torch.load(glob(f'../input/efficientdet/efficientdet_d{CNF[\"efd_level\"]}*.pth')[0]) \n        self.net.load_state_dict(checkpoint)\n        self.config.num_classes = 1\n        self.config.image_size = CNF['nnimSize']\n        self.net.class_net = HeadNet(self.config, num_outputs=self.config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n        \n    def forward(self, x):\n        \n        return self.net(x)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_net(isTrn=False):\n    \n    net = GWDModel(isTrn=isTrn)\n    \n    return DetBenchTrain(net, net.config) if isTrn else DetBenchEval(net, net.config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from effdet.anchors import Anchors, AnchorLabeler, generate_detections, MAX_DETECTION_POINTS\nfrom effdet.loss import DetectionLoss\n\n# from timm import create_model\n\n# class EfficientDet(nn.Module):\n\n#     def __init__(self, config, norm_kwargs=None, pretrained_backbone=True):\n#         super(EfficientDet, self).__init__()\n#         norm_kwargs = norm_kwargs or dict(eps=.001, momentum=.01)\n#         self.backbone = create_model(\n#             config.backbone_name, features_only=True, out_indices=(2, 3, 4),\n#             pretrained=pretrained_backbone, **config.backbone_args)\n#         feature_info = [dict(num_chs=f['num_chs'], reduction=f['reduction'])\n#                         for i, f in enumerate(self.backbone.feature_info())]\n#         self.fpn = BiFpn(config, feature_info, norm_kwargs=norm_kwargs)\n#         self.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=norm_kwargs)\n#         self.box_net = HeadNet(config, num_outputs=4, norm_kwargs=norm_kwargs)\n\n#         for n, m in self.named_modules():\n#             if 'backbone' not in n:\n#                 _init_weight(m, n)\n\n#     def forward(self, x):\n#         x = self.backbone(x)\n#         x = self.fpn(x)\n#         x_class = self.class_net(x)\n#         x_box = self.box_net(x)\n#         return x_class, x_box    \n\n\nclass DetBenchTrain(nn.Module):\n    def __init__(self, model, config):\n        super(DetBenchTrain, self).__init__()\n        self.config = config;self.model = model\n        anchors = Anchors(config.min_level, config.max_level,config.num_scales, config.aspect_ratios,config.anchor_scale, config.image_size)\n        self.anchor_labeler = AnchorLabeler(anchors, config.num_classes, match_threshold=0.5)\n        self.loss_fn = DetectionLoss(self.config)\n\n    def forward(self, x, gt_boxes, gt_labels, num_mixup=0):\n                \n        # 追加するパターン\n        for i in range(  num_mixup ):\n\n            x = torch.cat([x, (0.5*x[2*i,:,:,:]+0.5*x[2*i+1,:,:,:]).unsqueeze(0)])\n            gt_boxes.append( torch.cat([gt_boxes[2*i] ,gt_boxes[2*i+1]] ) )\n            gt_labels.append(torch.cat([gt_labels[2*i],gt_labels[2*i+1]]) )\n\n        class_out, box_out = self.model(x)\n\n        cls_targets = []\n        box_targets = []\n        num_positives = []\n        # FIXME this may be a bottleneck, would be faster if batched, or should be done in loader/dataset?\n        for i in range(x.shape[0]):\n            gt_class_out, gt_box_out, num_positive = self.anchor_labeler.label_anchors(gt_boxes[i], gt_labels[i])\n            cls_targets.append(gt_class_out)\n            box_targets.append(gt_box_out)\n            num_positives.append(num_positive)\n\n        return self.loss_fn(class_out, box_out, cls_targets, box_targets, num_positives)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_experiment(marking):\n    \n    marking = append_folds(marking,is_jigsaw_group_split=CNF['train_jigsaw'])\n    marking = marking.dropna() # jigsawしてない画像は「今回は外す、考慮しない、全部jigsawできた」とする\n    \n    # subsample\n    if CNF['frac']!=1:\n        image_list = pd.DataFrame(marking.image_id.unique()).sample(frac=CNF['frac'],random_state=42).values.squeeze()\n        marking = marking[marking.image_id.isin(image_list)]\n\n    \n    train_dataset, validation_dataset, test_dataset = createDataSet(marking)\n    \n    net = get_net(isTrn=True)\n    \n    if CNF['balance_sampling']:\n        \n        _tmp=marking[['merge_id','fold','trn_weight']].drop_duplicates()\n        trn_sample_weight=_tmp.loc[_tmp['fold']!=0,'trn_weight']\n    \n    else:\n        trn_sample_weight=None\n    \n    train_history=run_training(net, train_dataset, validation_dataset,test_dataset,trn_sample_weight)\n    \n    _save(train_history,f'history_{CNF[\"id\"]}.pkl')\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    ax.plot(np.array(train_history['trn_loss']),label='trn',color='blue')\n    ax.plot(np.array(train_history['val_loss']),label='val',color='red')\n    ax.legend()\n    title_txt=str(CNF).translate(str.maketrans({\"'\":\"\",\"}\":\"\",\"{\":\"\"}))\n    ax.set_title(title_txt);ax.set_ylim(0, 1)\n    return train_history\n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Experiment","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# if CNF['train']:\n\n#     result_df=pd.DataFrame(np.zeros((1,1)),columns=['experiment'],index=['loss'])\n\n#     history=run_experiment(marking.copy())\n    \n#     result_df.loc['loss','experiment'] = history['val_loss'][-1]#history['tst_loss'][-1]\n#     result_df.to_csv('result_df.csv',index=False)\n#     for ep,v in enumerate(history['val_loss']):\n#         print(f\"EP{ep:03d}:{v:.5f}\")\n#     result_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Check Predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# if CNF['checkPredcitions']:\n\n#     marking = append_folds(marking,is_jigsaw_group_split=CNF['train_jigsaw'])\n#     marking = marking.dropna() # jigsawしてない画像は「今回は外す、考慮しない、全部jigsawできた」とする\n    \n#     # 外部データ入れるとバグあるから応急処置\n#     marking=marking[marking.source!='ex01a']\n\n#     train_dataset, validation_dataset, test_dataset = createDataSet(marking)\n\n#     val_loader = torch.utils.data.DataLoader(validation_dataset, batch_size=TrainGlobalConfig.batch_size,num_workers=0, #num_workers=0じゃないとkaggleではおちる\n#             shuffle=False,sampler=SequentialSampler(validation_dataset),pin_memory=False,collate_fn=collate_fn,)\n\n#     best_ckpt_path=glob(CNF['bestCKPT'])[0] if 'bestCKPT' in CNF.keys() else glob(f\"effdet-*/*best*\")[0]\n    \n#     # load ckpt ,make prediction\n#     net = get_net(isTrn=False)\n#     net.to(device)\n#     fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)\n#     fitter.load(best_ckpt_path)\n#     raw_pred=fitter.predict(val_loader,score_threshold=0.5)\n#     iou_thresholds = [x for x in np.arange(0.5, 0.76, 0.05)]\n#     scores=[p['scores'] for p in raw_pred]\n#     # reshape pred/gt boxes \n#     amp = 1024/CNF['nnimSize']\n#     pred_boxes=[ ((amp*p['boxes']).astype(np.int32).clip(min=0, max=1023))[p['scores'].argsort()[::-1]] for p in raw_pred ]\n#     gt_boxes=_flattenList([[ _swapXY((amp*tgt['boxes']).cpu().numpy()).astype(np.int32) for tgt in val_batch[1] ] for val_batch in list(val_loader)])\n#     scores=[p['scores'] for p in raw_pred]\n\n#     from wheatutil import calculate_image_precision\n#     metrics=np.stack([ np.array(calculate_image_precision(gt_box,pred_box, thresholds = iou_thresholds, form = 'pascal_voc')) for gt_box,pred_box in zip(gt_boxes,pred_boxes) ])\n#     metrics={'cmp_scores':metrics[:,0],'precisions':metrics[:,1],'recalls':metrics[:,2] }\n\n#     fig, ax = plt.subplots(1, 3, figsize=(15, 3))\n#     for idx,metricName in enumerate([ 'cmp_scores','precisions','recalls' ]):\n\n#         ax[idx].hist(metrics[metricName],bins=50)\n#         ax[idx].axvline(np.mean(metrics[metricName]),c='r');\n#         ax[idx].set_title(f\"MEAN {metricName} {np.mean(metrics[metricName]):.5f}\");\n\n#     metrics.update({'image_id':[ _d[2] for _d in validation_dataset ]})\n#     df=pd.DataFrame(metrics).merge(marking,how='left',on='image_id').drop(columns=['fold'])\n#     _save(metrics,f\"validation_score_{CNF['id']}.pkl\")\n#     print(df.groupby('source').mean()[['cmp_scores','precisions','recalls']])\n    \n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['checkPredcitions']:\n    print(f\"{np.mean(metrics['cmp_scores']):.4f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['checkPredcitions']:\n    \n    NCOL=4\n    fig, ax = plt.subplots(9, NCOL, figsize=(18, 12*NCOL))\n    plt.subplots_adjust(left=0.005,right=0.995,bottom=0.005,top=0.995,wspace=0.005,hspace=0.005)\n    for metric_idx,metricName in enumerate([ 'cmp_scores','precisions','recalls' ]):\n\n        dataIds={}\n        dataIds['low'] = metrics[metricName].argsort()[:NCOL]\n        dataIds['mid'] = metrics[metricName].argsort()[len(metrics[metricName])//2:(NCOL+len(metrics[metricName])//2)]\n        dataIds['high']= metrics[metricName].argsort()[-NCOL:][::-1]\n\n        for quality_idx,(qualityName,ids) in enumerate(dataIds.items()):\n\n            for col_idx,image_id in enumerate(ids):\n                image=validation_dataset[image_id][0].permute(1,2,0).cpu().numpy()\n                for box in (gt_boxes[image_id]/amp).astype(np.int32):\n                    cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), (1, 1, 1), 6) \n                for box_idx,box in enumerate((pred_boxes[image_id]/amp).astype(np.int32)):\n                    cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), (1, 0, 0), (scores[image_id][box_idx]>iou_thresholds).sum()) \n\n                ax[metric_idx*3+quality_idx][col_idx].imshow(image)\n                ax[metric_idx*3+quality_idx][col_idx].set_axis_off()\n                ax[metric_idx*3+quality_idx][col_idx].set_title(f\"{metricName}-{qualityName}\")\n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_predictions(images,net, score_threshold=0.22):\n    images = torch.stack(images).cuda().float()\n    predictions = []\n    with torch.no_grad():\n        det = net(images, torch.tensor([1]*images.shape[0]).float().cuda())\n        for i in range(images.shape[0]):\n            boxes = det[i].detach().cpu().numpy()[:,:4]    \n            scores = det[i].detach().cpu().numpy()[:,4]\n            indexes = np.where(scores > score_threshold)[0]\n            boxes = boxes[indexes]\n            boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n            predictions.append({\n                'boxes': boxes[indexes],\n                'scores': scores[indexes],\n            })\n    return [predictions]\n\ndef run_wbf(predictions, image_index, image_size=CNF['nnimSize'], iou_thr=0.44, skip_box_thr=0.43, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist()  for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist()  for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).tolist() for prediction in predictions]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def infer():\n    \n    test_df=pd.DataFrame([path.split('/')[-1][:-4] for path in glob(f\"{IMAGE_ROOT_PATH}/*jpg\")],columns=['image_id'])\n    test_dataset = DatasetRetriever(jigsaw=False,marking=test_df,transforms=get_test_transforms(),test=True,)\n    tst_loader = DataLoader(test_dataset,batch_size=2,shuffle=False,num_workers=0,drop_last=False,collate_fn=collate_fn)\n    net = get_net(isTrn=False).to(device)\n    best_ckpt_path=glob(CNF['bestCKPT'])[0] if 'bestCKPT' in CNF.keys() else glob(f\"effdet-*/*best*\")[0]\n    checkpoint = torch.load(best_ckpt_path, map_location=device) # GPUの時にチェックしてない\n    net.model.load_state_dict(checkpoint['model_state_dict'])\n\n    for j, (images, image_ids) in enumerate(tst_loader):\n        break\n\n    predictions = make_predictions(images,net)\n\n    i = 0\n    sample = images[i].permute(1,2,0).cpu().numpy()\n\n    boxes, scores, labels = run_wbf(predictions, image_index=i)\n    boxes = boxes.astype(np.int32).clip(min=0, max=511)\n\n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for box in boxes:\n        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (1, 0, 0), 1)\n\n    ax.set_axis_off()\n    ax.imshow(sample);\n    \n    \n    results = []\n    ret_boxes = {};ret_scores = {}\n    amp = 1024/CNF['nnimSize']\n    \n    for images, image_ids in tst_loader:\n        predictions = make_predictions(images,net)\n        for i, image in enumerate(images):\n            boxes, scores, labels = run_wbf(predictions, image_index=i)\n            boxes = (boxes*amp).astype(np.int32).clip(min=0, max=1023)\n            image_id = image_ids[i]\n\n            \n            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n\n            result = {\n                'image_id': image_id,\n                'PredictionString': format_prediction_string(boxes, scores),\n            }\n            results.append(result)\n            ret_boxes[image_id]=boxes\n            ret_scores[image_id]=scores\n\n    test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])\n    \n    return test_df,ret_boxes,ret_scores\n    \n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['infer']:\n    \n    # global name IMAGE_ROOT_PATH\n    IMAGE_ROOT_PATH =  '../input/global-wheat-detection/test'\n    \n    test_df,_,_ = infer()#['image_id', 'PredictionString']\n    test_df.to_csv('submission.csv', index=False)\n    test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CNF['createPseudo']:\n    \n    # global name IMAGE_ROOT_PATH\n    #IMAGE_ROOT_PATH =  '../input/external01a/external01a'\n    IMAGE_ROOT_PATH =  '../input/global-wheat-detection/train'\n    \n    pseudo_df, bboxes, scores = infer()\n    \n    _df=[]\n    for image_id in bboxes.keys():\n\n        _tmp_df = pd.DataFrame([ f\"[{box[0]},{box[1]},{box[2]},{box[3]}]\" for box in bboxes[image_id] ],columns=['bbox'])\n        _tmp_df['image_id'] = image_id\n        _tmp_df['score'] = scores[image_id]\n        _df.append(_tmp_df)\n\n    pseudo_df = pd.concat(_df)\n    pseudo_df['width']=1024\n    pseudo_df['height']=1024\n    pseudo_df['source']='ex01b'\n    #pseudo_df=pseudo_df[['image_id','width','height','bbox','source']]\n    pseudo_df=pseudo_df[['image_id','width','height','bbox']]\n    pseudo_df.to_csv(f\"pred_bbox_{CNF['id']}.csv\",index=False)\n    \n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if False:\n    !mkdir data\n    for _fold in range(2):\n        !mkdir data/fold{_fold}\n        for _src in SRC_IDS:\n            !mkdir data/fold{_fold}/{_src}\n\n            for image_id in df_folds[(df_folds.fold==_fold)&(df_folds.source==_src)].index:\n\n                print(image_id)\n\n                !cp {IMAGE_ROOT_PATH}/{image_id}.jpg data/fold{_fold}/{_src}/\n            \n        ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}