{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"! nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%cd ../input/effdetapex/apex\n!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./ \n%cd ../","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-deps '/kaggle/input/efficientdetpytorch/efficientdet-pytorch/timm-0.1.30-py3-none-any.whl' > /dev/null\n!pip install --no-deps '/kaggle/input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null\n!pip install --no-deps '/kaggle/input/efficientdetpytorch/efficientdet-pytorch/omegaconf-2.0.0-py3-none-any.whl' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dependencies and imports","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1,2,3\"\nimport json\nimport zipfile\n# import gluoncv as gcv\nimport torch\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport numba\nimport ast\nfrom numba import jit\nfrom typing import List, Union, Tuple\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nfrom glob import glob\nimport logging\nimport math\nimport copy\nfrom tqdm import tqdm\nfrom timm.utils import *\nfrom timm.optim import create_optimizer\nfrom timm.scheduler import create_scheduler\nfrom timm.data.distributed_sampler import OrderedDistributedSampler\nfrom tqdm import tqdm\n\nimport sys\nsys.path.insert(0, '/kaggle/input/efficientdetpytorch/efficientdet-pytorch')\nsys.path.insert(0, '/kaggle/input/weightedboxesfusion')\n\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchTrain,DetBenchPredict\nfrom effdet.efficientdet import HeadNet\nimport ensemble_boxes\nfrom ensemble_boxes import *\nimport gc\nimport re\nimport matplotlib.pyplot as plt\nfrom itertools import product\ntry:\n    from apex import amp\n    from apex.parallel import DistributedDataParallel as DDP\n    from apex.parallel import convert_syncbn_model\n    has_apex = True\nexcept ImportError:\n    from torch.nn.parallel import DistributedDataParallel as DDP\n    has_apex = False\n\n# define seed and image size\nSEED = 42\nAREA_SMALL = 56 * 56\nIMG_SIZE = 1024\nRESIZE_IMG_SIZE = 1024\nNUM_CLASSES = 1\nIMAGENET_DEFAULT_MEAN = [x * 255 for x in (0.485, 0.456, 0.406)]\nIMAGENET_DEFAULT_STD = [x * 255 for x in (0.229, 0.224, 0.225)]\nTRAIN_IMAGE_PATH = '/kaggle/input/global-wheat-detection/train'\nTEST_IMAGE_PATH = '/kaggle/input/global-wheat-detection/test'\nTRAIN_CSV_PATH = '/kaggle/input/global-wheat-detection/train.csv'\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## GlobalConfig","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class GlobalConfig:\n    num_workers = 2\n    pseudo_train_batch_size = 2\n    test_batch_size = 1\n    n_epochs = 5 # n_epochs = 40\n    lr = 1e-4\n    warmup_epoch = 0\n    checkpoint = False\n    weight_decay = 4e-5\n    eps = 1e-3\n    nbs = 64\n    folder = '/kaggle/working/effdet5-1024-pseudo'\n    \n    # multi-gpu training\n    distributed = False\n    \n    verbose = True\n    verbose_step = 1\n \n    step_scheduler = True  # do scheduler.step after optimizer.step\n    validation_scheduler = False  # do scheduler.step after validation stage loss\n\n    SchedulerClass = torch.optim.lr_scheduler.CosineAnnealingLR\n    \n    checkpoint_path = '/kaggle/input/efficientded/effdet5-1024-last.bin'\n    config_name = 'tf_efficientdet_d5'\n    submission_path = '/kaggle/working/submission.csv'\n    final_checkpoint_path='/kaggle/working/effdet5-1024-pseudo/best-checkpoint.bin'\n    \n    tta_score_thr = 0.05\n    tta_wbf_iou_thr = 0.34\n    tta_wbf_skip_box_thr =0.42\n    \n    \n    final_tta_score_thr = 0.05\n    final_tta_wbf_iou_thr = 0.34\n    final_tta_wbf_skip_box_thr =0.48\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n    \n\ndef get_train_transforms():\n    return A.Compose(\n        [\n            A.RandomSizedCrop(min_max_height=(800, 800), height=IMG_SIZE, width=IMG_SIZE, p=0.5),\n            A.OneOf([\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, \n                                     val_shift_limit=0.2, p=0.9),\n                A.RandomBrightnessContrast(brightness_limit=0.2, \n                                           contrast_limit=0.2, p=0.9),\n            ],p=0.9),\n            A.ToGray(p=0.01),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=RESIZE_IMG_SIZE, width=RESIZE_IMG_SIZE, p=1),\n            A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),\n        ], \n        p=1.0,\n        bbox_params=A.BboxParams(\n            format='pascal_voc',\n            min_area=0, \n            min_visibility=0,\n            label_fields=['labels']\n        )\n    )\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.Resize(height=RESIZE_IMG_SIZE, width=RESIZE_IMG_SIZE, p=1.0),\n            ToTensorV2(p=1.0)], \n            p=1.0)\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], \n                                                             j[1][2], j[1][3]))\n    return \" \".join(pred_strings)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, transforms=None, test=False, pseudo_label=False):\n        super().__init__()\n\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.pseudo_label = pseudo_label\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        p_ratio = random.random()\n        if self.pseudo_label:\n            image,boxes = self.load_mixup_image_and_boxes(index)\n        else:\n            if self.test or p_ratio >0.5 :\n                image, boxes = self.load_image_and_boxes(index)\n            else:\n                image, boxes = self.load_cutmix_image_and_boxes(index)\n\n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['img_scale'] = torch.tensor([1.])\n        target['image_id'] = torch.tensor([index])\n        target['img_size'] = torch.tensor([(IMG_SIZE, IMG_SIZE)])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                sample['bboxes'] = self.boxes_clean(sample['bboxes'])\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        if self.pseudo_label:\n            image = cv2.imread(f'{TEST_IMAGE_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        else:\n            image = cv2.imread(f'{TRAIN_IMAGE_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        if image.shape[0]!=IMG_SIZE or image.shape[1]!=IMG_SIZE:\n            image = cv2.resize(image,(IMG_SIZE,IMG_SIZE))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return image, boxes.astype(np.int32)\n\n    def load_cutmix_image_and_boxes(self, index, imsize=IMG_SIZE):\n        \"\"\" \n        This implementation of cutmix author:  https://www.kaggle.com/nvnnghia \n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize\n        s = imsize // 2\n    \n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y\n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]\n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32)\n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)\n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n\n            boxes[:, 0] += padw\n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n        if len(result_boxes):\n            result_boxes = np.concatenate(result_boxes, 0)\n            np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[np.where((result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1]) > 0)]\n        return result_image, result_boxes\n    \n \n    def load_mixup_image_and_boxes(self, index):\n        image, boxes = self.load_image_and_boxes(index)\n        r_image, r_boxes = self.load_image_and_boxes(random.randint(0, self.image_ids.shape[0] - 1))\n        return (image+r_image)/2, np.vstack((boxes, r_boxes)).astype(np.int32)\n    \n    def boxes_clean(self,list_boxes):\n        new_list_boxes = []\n        for box in list_boxes:\n            box_w = box[2]-box[0]\n            box_h = box[3]-box[1]\n            box_area = (box[2]-box[0])*(box[3]-box[1])\n            box_wh_ratio = (box[2]-box[0])/(box[3]-box[1]+1e-16)\n            box_hw_ratio = (box[3]-box[1])/(box[2]-box[0]+1e-16)\n            if box_w>10.0 and box_h>10.0 and box_area> 300.0 and box_wh_ratio<10.0 and box_hw_ratio<10.0:\n                new_list_boxes.append(box)\n        return new_list_boxes\n\n    \nclass DatasetRetriever_test(Dataset):\n\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TEST_IMAGE_PATH}/{image_id}', cv2.IMREAD_COLOR)\n        if image.shape[0]!=IMG_SIZE or image.shape[1]!=IMG_SIZE:\n            image = cv2.resize(image,(IMG_SIZE,IMG_SIZE))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        \n        target = {}\n        target['img_scale'] = torch.tensor([1.])\n        \n        return image, image_id, target\n\n    def __len__(self) -> int:\n        return len(self.image_ids)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Detection Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_net(checkpoint_path, config_name, predict=True):\n    config = get_efficientdet_config(config_name)\n    net = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = NUM_CLASSES\n    config.image_size = RESIZE_IMG_SIZE\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n    if os.path.isfile(checkpoint_path):\n        checkpoint = torch.load(checkpoint_path)\n        net.load_state_dict(checkpoint['model_state_dict'])\n    else:\n        checkpoint = torch.load(GlobalConfig.checkpoint_path)\n        net.load_state_dict(checkpoint['model_state_dict'])\n\n    del checkpoint\n    gc.collect()\n    if predict:\n        net = DetBenchPredict(net, config)\n    else:\n        net = DetBenchTrain(net, config)\n    return net\n\n\ndef run_wbf(predictions, image_index, image_size, iou_thr, skip_box_thr, weights=None):\n    boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]\n    scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]\n    labels = [np.ones(prediction[image_index]['scores'].shape[0]).astype(int).tolist() for prediction in predictions]\n    boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\ndef TTA_Compose():\n    tta_transforms = []\n    for tta_combination in product([TTAHorizontalFlip(), None], \n                               [TTAVerticalFlip(), None],\n                               [TTARotate90(), None]):\n        tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))\n    return tta_transforms\n\ndef make_tta_predictions(images,targets,net,score_threshold):\n    with torch.no_grad():\n        images = torch.stack(images).float().cuda()\n        img_scale = torch.tensor([target['img_scale'].cuda() for target in targets])\n        img_size = torch.tensor([(RESIZE_IMG_SIZE, RESIZE_IMG_SIZE) for target in targets]).cuda()\n        tta_transforms = TTA_Compose()\n        net = net.eval()\n        net = net.cuda()\n        predictions = []\n        for tta_transform in tta_transforms:\n            result = []\n            det = net(tta_transform.batch_augment(images.clone()),\n                      img_scales = img_scale,\n                      img_size = img_size)\n            for i in range(images.shape[0]):\n                boxes = det[i].detach().cpu().numpy()[:,:4]    \n                scores = det[i].detach().cpu().numpy()[:,4]\n                indexes = np.where(scores > score_threshold)[0]\n                boxes = boxes[indexes]\n                boxes[:, 2] = boxes[:, 2] + boxes[:, 0]\n                boxes[:, 3] = boxes[:, 3] + boxes[:, 1]\n                boxes = tta_transform.deaugment_boxes(boxes.copy())\n                result.append({\n                    'boxes': boxes,\n                    'scores': scores[indexes],\n                })\n            predictions.append(result)\n    return predictions\n\n\n\ndef detect_images(data_loader, checkpoint_path, config_name, score_threshold, iou_thr, skip_box_thr):\n    net = get_net(checkpoint_path, config_name, predict=True)\n    results = []\n    all_boxes = []\n    all_scores = []\n    all_image_ids = []\n    pseudo_labels = []\n    for images, image_ids, targets in data_loader:\n        predictions = make_tta_predictions(images,targets,net,score_threshold)\n        for i, image in enumerate(images):\n            boxes, scores, labels = run_wbf(predictions, i, RESIZE_IMG_SIZE, iou_thr, skip_box_thr)\n            boxes = (boxes*float(1024/1024)).round().astype(np.int32).clip(min=0, max=RESIZE_IMG_SIZE-1)\n            \n            image_id = image_ids[i]\n        \n            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n            \n            result = {\n                'image_id': image_id[:-4],\n                'PredictionString': format_prediction_string(boxes, scores)\n            }\n            \n            for box in boxes:\n                pseudo_label = {\n                    'image_id': image_id[:-4],\n                    'source': 'Libo',\n                    'width':RESIZE_IMG_SIZE,\n                    'height':RESIZE_IMG_SIZE,\n                    'x': box[0],\n                    'y': box[1],\n                    'w': box[2],\n                    'h': box[3]        \n                }\n                pseudo_labels.append(pseudo_label)\n                \n            results.append(result)\n            all_boxes.append(boxes)\n            all_scores.append(scores)\n            all_image_ids.append(image_id)\n    pseudo_labels_df = pd.DataFrame(pseudo_labels, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])\n    return all_image_ids, all_boxes, all_scores, results, pseudo_labels_df\n   \n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_rect_with_score(img, bboxes, scores,color=None):\n    img = img.copy()\n    bboxes = bboxes[:, :4]\n    bboxes = bboxes.reshape(-1, 4)\n    for bbox,score in zip(bboxes,scores):\n        pt1, pt2 = (bbox[0], bbox[1]), (bbox[2], bbox[3])\n        pt1 = int(pt1[0]), int(pt1[1])\n        pt2 = int(pt2[0]), int(pt2[1])\n        img = cv2.rectangle(img.copy(), pt1, pt2, color, int(max(img.shape[:2]) / 200))\n        cv2.putText(img, '%.2f'%(score), pt1, cv2.FONT_HERSHEY_SIMPLEX,1, (255,255,255), 2, cv2.LINE_AA)                 \n    return img\n\ndef draw_test_image(all_image_ids, all_boxes, all_scores, nums):\n    fig, ax = plt.subplots(nums, 1, figsize=(120, 60))\n    for i in range(nums):\n        im0 = cv2.imread(f'{TEST_IMAGE_PATH}/{all_image_ids[i]}')[:,:,::-1]\n        box0 = all_boxes[i]\n        box0[:,2] = box0[:,2]+box0[:,0]\n        box0[:,3] = box0[:,1]+box0[:,3]\n        box0 = box0.clip(min=0,max=RESIZE_IMG_SIZE-1)\n        score0 = np.array(all_scores[i])\n        img = draw_rect_with_score(im0,np.array(box0),score0, color=(255,0,0))\n        ax[i].imshow(img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split Data and Create Dataloader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_train_csv(path):\n    marking = pd.read_csv(path)\n    bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n    for i, column in enumerate(['x', 'y', 'w', 'h']):\n        marking[column] = bboxs[:,i]\n    marking.drop(columns=['bbox'], inplace=True)\n    return marking\n\ndef df_pre(marking,ispseudo=False):\n    marking['w'] = marking['w'].astype(np.float32)\n    marking['h'] = marking['h'].astype(np.float32)\n    marking['wh_ratio'] = marking['w'] / (marking['h']+1e-16)\n    marking['hw_ratio'] = marking['h'] / (marking['w']+1e-16)\n    marking['area'] = marking['w'] * marking['h']\n    if not ispseudo:\n        bad_boxes = [3687,117344,173,113947,52868,2159,2169,121633,121634,147504,118211,52727,147552]\n        marking = marking[~marking.index.isin(bad_boxes)]\n        error_bbox = [100648.0, 145360.0, 149744.0, 119790.0, 106743.0]\n        marking = marking[~marking['area'].isin(error_bbox)]\n        marking = marking[marking['area']<154200.0]\n    marking = marking[marking['area']>300.0]\n    marking = marking[marking['w']>10.0]\n    marking = marking[marking['h']>10.0]\n    marking = marking[marking['wh_ratio']<10]\n    marking = marking[marking['hw_ratio']<10]\n    if not ispseudo:\n        marking = marking.drop([4775,111973,113007,115239,40664,46030])\n    return marking\n\ndef dataset_split(train_df,pseudo_df,sk_n,fold_n,pre=True):\n    if pre:\n        train_df_pre = df_pre(train_df,False)\n        #pseudo_df_pre = df_pre(pseudo_df,True)\n        pseudo_df_pre = pseudo_df\n    else:\n        train_df_pre = train_df\n        pseudo_df_pre = pseudo_df\n        \n    skf = StratifiedKFold(n_splits=sk_n, shuffle=True, random_state=42)\n    df_folds = train_df_pre[['image_id']].copy()\n    df_folds.loc[:, 'bbox_count'] = 1\n    df_folds = df_folds.groupby('image_id').count()\n    df_folds.loc[:, 'source'] = train_df_pre[['image_id', 'source']].groupby('image_id').min()['source']\n    df_folds.loc[:, 'stratify_group'] = np.char.add(\n        df_folds['source'].values.astype(str),\n        df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str))\n    df_folds.loc[:, 'fold'] = 0\n\n    for fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n        df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n    \n    train_dataset = DatasetRetriever(image_ids=df_folds[df_folds['fold'] != fold_n].index.values,\n                                     marking=train_df_pre,\n                                     transforms=get_train_transforms(),\n                                     test=False,\n                                     pseudo_label=False)\n    \n    validation_dataset = DatasetRetriever(image_ids=df_folds[df_folds['fold'] == fold_n].index.values,\n                                          marking=train_df_pre,\n                                          transforms=get_valid_transforms(),\n                                          test=True,\n                                          pseudo_label=False)\n    \n    pseudo_dataset = DatasetRetriever(image_ids=pseudo_df_pre['image_id'].unique(),\n                                       marking=pseudo_df_pre,\n                                       transforms=get_train_transforms(),\n                                       test=False,\n                                       pseudo_label=True)\n    \n    return train_dataset, validation_dataset, pseudo_dataset\n\n\n# for i in range(10,20):\n#     image, target, image_id = train_dataset_all[i]\n#     boxes = target['boxes'].cpu().numpy().astype(np.int32)\n\n#     numpy_image = image.permute(1,2,0).cpu().numpy()\n\n#     fig, ax = plt.subplots(1, 1, figsize=(12, 6))\n\n#     for box in boxes:\n#         cv2.rectangle(numpy_image, (box[1], box[0]), (box[3],  box[2]), (0, 1, 0), 2)\n    \n#     ax.set_axis_off()\n#     ax.imshow(numpy_image);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pseudo Label Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ModelEMA:\n    \"\"\" Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models\n    Keep a moving average of everything in the model state_dict (parameters and buffers).\n    This is intended to allow functionality like\n    https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n    A smoothed version of the weights is necessary for some training schemes to perform well.\n    E.g. Google's hyper-params for training MNASNet, MobileNet-V3, EfficientNet, etc that use\n    RMSprop with a short 2.4-3 epoch decay period and slow LR decay rate of .96-.99 requires EMA\n    smoothing of weights to match results. Pay attention to the decay constant you are using\n    relative to your update count per epoch.\n    To keep EMA from using GPU resources, set device='cpu'. This will save a bit of memory but\n    disable validation of the EMA weights. Validation will have to be done manually in a separate\n    process, or after the training stops converging.\n    This class is sensitive where it is initialized in the sequence of model init,\n    GPU assignment and distributed training wrappers.\n    I've tested with the sequence in my own train.py for torch.DataParallel, apex.DDP, and single-GPU.\n    \"\"\"\n\n    def __init__(self, model, decay=0.9999, device=''):\n        # Create EMA\n        self.ema = copy.deepcopy(model)  # FP32 EMA\n        self.ema.eval()\n        self.updates = 0  # number of EMA updates\n        self.decay = lambda x: decay * (1 - math.exp(-x / 2000))  # decay exponential ramp (to help early epochs)\n        self.device = device  # perform ema on different device from model if set\n        if device:\n            self.ema.to(device)\n        for p in self.ema.parameters():\n            p.requires_grad_(False)\n\n    def update(self, model):\n        # Update EMA parameters\n        with torch.no_grad():\n            self.updates += 1\n            d = self.decay(self.updates)\n\n            msd = model.state_dict()  # model state_dict\n            for k, v in self.ema.state_dict().items():\n                if v.dtype.is_floating_point:\n                    v *= d\n                    v += (1. - d) * msd[k].detach()\n\n    def update_attr(self, model):\n        # Update EMA attributes\n        for k, v in model.__dict__.items():\n            if not k.startswith('_') and k not in [\"process_group\", \"reducer\"]:\n                setattr(self.ema, k, v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nclass Fitter:\n    \n    def __init__(self, model, config, train_loader, val_loader):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'{config.folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n        \n        self.train_loader = train_loader\n        self.val_loader = val_loader\n        \n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n        \n        self.device = torch.device('cuda')\n        self.model = model.to(self.device)\n        #self.model = model.cuda()\n\n        self.nbs = self.config.nbs  # nominal batch size\n        self.accumulate = max(round(self.nbs / self.config.pseudo_train_batch_size), 1)\n\n        self.nb = len(self.train_loader)\n        self.n_burn = max(self.config.warmup_epoch*self.nb, 1e3)\n\n        self.config.weight_decay /= self.config.lr\n        parameters = self.add_weight_decay(self.model, self.config.weight_decay)\n        self.config.weight_decay = 0.\n        self.optimizer = torch.optim.AdamW(parameters, self.config.lr, weight_decay=self.config.weight_decay, eps=self.config.eps)\n        self.scheduler = config.SchedulerClass(self.optimizer, T_max=(self.config.n_epochs-self.config.warmup_epoch), eta_min=self.config.lr*1e-3)\n              \n        if has_apex:\n            print('Using apex')\n            self.model.model, self.optimizer = amp.initialize(self.model.model, self.optimizer, opt_level=\"O1\", verbosity=0)\n            if self.config.distributed:\n                print('Using DataParallel ,GPU nums:',torch.cuda.device_count())\n                self.model.model = torch.nn.DataParallel(self.model.model)\n        \n        if self.config.checkpoint:\n            self.load(f'{self.config.folder}/last-checkpoint.bin')\n        self.ema = ModelEMA(self.model.model)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n        self.do_scheduler = True\n    \n    def add_weight_decay(self, model, weight_decay=1e-5, skip_list=()):\n        decay = []\n        no_decay = []\n        for name, param in model.named_parameters():\n            if not param.requires_grad:\n                continue  # frozen weights\n            if len(param.shape) == 1 or name.endswith(\".bias\") or name in skip_list:\n                no_decay.append(param)\n            else:\n                decay.append(param)\n        return [\n            {'params': no_decay, 'weight_decay': 0.},\n            {'params': decay, 'weight_decay': weight_decay}]\n\n    def fit(self):\n        for epoch in range(self.epoch, self.config.n_epochs):\n            if epoch < self.config.warmup_epoch:\n                lr_scale = min(1., float(epoch+1) / float(self.config.warmup_epoch))\n                for pg in self.optimizer.param_groups:\n                    pg['lr'] = lr_scale * self.config.lr\n                self.do_scheduler = False\n            else: \n                self.do_scheduler = True\n\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch()\n            \n            if self.config.step_scheduler and self.do_scheduler:\n                self.scheduler.step()\n\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n            t = time.time()\n            summary_loss = self.validation()\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint.bin')\n\n            if self.config.validation_scheduler and self.do_scheduler:\n                self.scheduler.step()\n            self.ema.update_attr(self.model.model)\n\n            self.epoch += 1\n\n\n\n    def validation(self):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        t = time.time()\n        val_loader = tqdm(self.val_loader, total=len(self.val_loader), desc=\"Validating\")\n        for step, (images, targets, image_ids) in enumerate(val_loader):\n            with torch.no_grad():\n                pred_res = {}\n                images = torch.stack(images)\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                boxes = [target['boxes'].to(self.device).float() for target in targets]\n                labels = [target['labels'].to(self.device).float() for target in targets]\n                img_scale = torch.tensor([target['img_scale'].to(self.device) for target in targets])\n                img_size = torch.tensor([(IMG_SIZE, IMG_SIZE) for target in targets]).to(self.device).float()\n                \n                pred_res['bbox'] = boxes\n                pred_res['cls'] = labels\n                pred_res['img_scale'] = img_scale\n                pred_res['img_size'] = img_size\n\n                outputs = self.model(images, pred_res)\n                loss = outputs['loss']\n                \n                summary_loss.update(loss.detach().item(), batch_size)\n                val_loader.set_description(f'Valid Step {step}/{len(val_loader)}, ' + \\\n                            f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                            f'time: {(time.time() - t):.5f}')\n        return summary_loss\n\n    def train_one_epoch(self):\n        self.model.train()\n        summary_loss = AverageMeter()\n        t = time.time()\n        train_loader = tqdm(self.train_loader, total=len(self.train_loader), desc=\"Training\")\n        for step, (images, targets, image_ids) in enumerate(train_loader):\n            target_res = {}\n            ni = step + self.nb * self.epoch\n            if ni <= self.n_burn:\n                xi = [0, self.n_burn]  # x interp\n                self.accumulate = max(1, np.interp(ni, xi, [1, self.nbs / self.config.pseudo_train_batch_size]).round())\n\n            images = torch.stack(images)\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n            boxes = [target['boxes'].to(self.device).float() for target in targets]\n            labels = [target['labels'].to(self.device).float() for target in targets]\n            img_scale = torch.tensor([target['img_scale'] for target in targets]).to(self.device).float()\n            img_size = torch.tensor([(IMG_SIZE, IMG_SIZE) for target in targets]).to(self.device).float()\n\n            target_res['bbox'] = boxes\n            target_res['cls'] = labels\n            target_res['img_scale'] = img_scale\n            target_res['img_size'] = img_size\n            \n            outputs = self.model(images, target_res)\n            loss = outputs['loss']\n\n            with amp.scale_loss(loss, self.optimizer) as scaled_loss:\n                scaled_loss.backward()\n            \n            summary_loss.update(loss.detach().item(), batch_size)\n\n            # Optimize\n            if ni % self.accumulate == 0:\n                self.optimizer.step()\n                self.optimizer.zero_grad()\n                self.ema.update(self.model.model)\n            \n            train_loader.set_description(f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'Learning rate {self.optimizer.param_groups[0][\"lr\"]}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}')\n\n        return summary_loss\n    \n    def save(self, path):\n        self.model.eval()\n        if self.config.distributed:\n            torch.save({\n                'model_state_dict': self.model.model.module.state_dict(),\n                'optimizer_state_dict': self.optimizer.state_dict(),\n                'scheduler_state_dict': self.scheduler.state_dict(),\n                'best_summary_loss': self.best_summary_loss,\n                'epoch': self.epoch,\n            }, path)\n        else:\n            torch.save({\n                'model_state_dict': self.model.model.state_dict(),\n                'optimizer_state_dict': self.optimizer.state_dict(),\n                'scheduler_state_dict': self.scheduler.state_dict(),\n                'best_summary_loss': self.best_summary_loss,\n                'epoch': self.epoch,\n            }, path)\n        print('Saved epoch',self.epoch)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        if self.config.distributed:\n            self.model.model.module.load_state_dict(checkpoint['model_state_dict'])\n            self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n            self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n            self.best_summary_loss = checkpoint['best_summary_loss']\n            self.epoch = checkpoint['epoch'] + 1\n        else:\n            self.model.model.load_state_dict(checkpoint['model_state_dict'])\n            self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n            self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n            self.best_summary_loss = checkpoint['best_summary_loss']\n            self.epoch = checkpoint['epoch'] + 1\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_training(train_loader,val_loader,checkpoint_path):\n    # create net\n    net = get_net(checkpoint_path, GlobalConfig.config_name, predict=False)\n    # pseudo label training\n    fitter = Fitter(model=net, config=GlobalConfig, train_loader=train_loader, val_loader=val_loader)\n    fitter.fit()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(os.listdir(TEST_IMAGE_PATH))\ntest_dataset = DatasetRetriever_test(\n    image_ids=os.listdir(TEST_IMAGE_PATH),\n    transforms=get_valid_transforms()\n)\ntest_dataloader = DataLoader(\n    test_dataset,\n    batch_size=GlobalConfig.test_batch_size,\n    shuffle=True,\n    num_workers=GlobalConfig.num_workers,\n    pin_memory=False,\n    drop_last=False,\n    collate_fn=collate_fn\n)\nall_image_ids,all_boxes,all_scores,results, pseudo_df = detect_images(test_dataloader,GlobalConfig.checkpoint_path, \n                                                                             GlobalConfig.config_name, \n                                                                             GlobalConfig.tta_score_thr, GlobalConfig.tta_wbf_iou_thr, \n                                                                             GlobalConfig.tta_wbf_skip_box_thr)\n\ndraw_test_image(all_image_ids, all_boxes, all_scores, 10)\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pseudo_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = format_train_csv(TRAIN_CSV_PATH)\n# train_df = train_df[:500]\ntrain_dataset, validation_dataset,pseudo_dataset = dataset_split(train_df,pseudo_df,5,0,True)\n#create dataloader\ntrain_dataset_all = train_dataset + pseudo_dataset\n# validation_dataset_all = validation_dataset+pseudo_dataset\ntrain_loader = torch.utils.data.DataLoader(\n               train_dataset_all,\n               batch_size=GlobalConfig.pseudo_train_batch_size,\n               sampler=RandomSampler(train_dataset_all),\n               pin_memory=False,\n               drop_last=False,\n               num_workers=GlobalConfig.num_workers,\n               collate_fn=collate_fn)\n\nval_loader = torch.utils.data.DataLoader(\n             validation_dataset, \n             batch_size=GlobalConfig.pseudo_train_batch_size,\n             num_workers=GlobalConfig.num_workers,\n             shuffle=False,\n             sampler = SequentialSampler(validation_dataset),\n             pin_memory=False,\n             collate_fn=collate_fn)\n# pseudo label training\nif len(os.listdir(TEST_IMAGE_PATH)) > 10:\n    run_training(train_loader,val_loader,GlobalConfig.checkpoint_path)\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Final Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"final_image_ids,final_boxes,final_scores,final_results, final_pseudo_df = detect_images(test_dataloader,GlobalConfig.final_checkpoint_path, \n                                                                                        GlobalConfig.config_name, \n                                                                                        GlobalConfig.final_tta_score_thr, \n                                                                                        GlobalConfig.final_tta_wbf_iou_thr, \n                                                                                        GlobalConfig.final_tta_wbf_skip_box_thr)\n\ndraw_test_image(final_image_ids,final_boxes,final_scores, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = format_train_csv(TRAIN_CSV_PATH)\n# # train_df = train_df[:500]\n# train_dataset, validation_dataset,pseudo_dataset = dataset_split(train_df,final_pseudo_df,5,0,True)\n# #create dataloader\n# train_dataset_all = train_dataset + pseudo_dataset\n# # validation_dataset_all = validation_dataset+pseudo_dataset\n# train_loader = torch.utils.data.DataLoader(\n#                train_dataset_all,\n#                batch_size=GlobalConfig.pseudo_train_batch_size,\n#                sampler=RandomSampler(train_dataset_all),\n#                pin_memory=False,\n#                drop_last=False,\n#                num_workers=GlobalConfig.num_workers,\n#                collate_fn=collate_fn)\n\n# val_loader = torch.utils.data.DataLoader(\n#              validation_dataset, \n#              batch_size=GlobalConfig.pseudo_train_batch_size,\n#              num_workers=GlobalConfig.num_workers,\n#              shuffle=False,\n#              sampler = SequentialSampler(validation_dataset),\n#              pin_memory=False,\n#              collate_fn=collate_fn)\n# # pseudo label training\n# if len(os.listdir(TEST_IMAGE_PATH)) > 10:\n#     run_training(train_loader,val_loader,GlobalConfig.final_checkpoint_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# final_image_ids,final_boxes,final_scores,final_results, final_pseudo_df = detect_images(test_dataloader,GlobalConfig.final_checkpoint_path, \n#                                                                                         GlobalConfig.config_name, \n#                                                                                         GlobalConfig.final_tta_score_thr, \n#                                                                                         GlobalConfig.final_tta_wbf_iou_thr, \n#                                                                                         GlobalConfig.final_tta_wbf_skip_box_thr)\n\n# draw_test_image(final_image_ids,final_boxes,final_scores, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_results = pd.DataFrame(final_results, columns=['image_id', 'PredictionString'])\nfinal_results.to_csv(GlobalConfig.submission_path, index=False)\nfinal_results.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## OOF","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}