{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp -r ../input/detrssoc/resources .","metadata":{"execution":{"iopub.status.busy":"2023-05-24T04:28:11.004103Z","iopub.execute_input":"2023-05-24T04:28:11.004479Z","iopub.status.idle":"2023-05-24T04:28:12.050592Z","shell.execute_reply.started":"2023-05-24T04:28:11.004451Z","shell.execute_reply":"2023-05-24T04:28:12.048771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport torch\n\n# call before train\nSEED = 42\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\n# dataset config\nDATA_DIR = '/kaggle/input/global-wheat-detection'\nTRAIN_ROOT_PATH = DATA_DIR + '/train'\n\n# train cache\nCACHE_DIR = '/kaggle/working'\n\ndef format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-24T04:28:12.057564Z","iopub.execute_input":"2023-05-24T04:28:12.061113Z","iopub.status.idle":"2023-05-24T04:28:12.079218Z","shell.execute_reply.started":"2023-05-24T04:28:12.061047Z","shell.execute_reply":"2023-05-24T04:28:12.078557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom sklearn.model_selection import StratifiedKFold\nimport cv2\nimport sys\nsys.path.append('./resources/')\nsys.path.append('./resources/detr')\n\n# to solve import error\nimport torchvision\n_version = torchvision.__version__\ntorchvision.__version__ = '0.8'\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n# to solve import error\ntorchvision.__version__ = _version\n\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom glob import glob\n\ndef get_train_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\n\ndef normalize_bbox(bbox, rows, cols):\n    \"\"\"Normalize coordinates of a bounding box. Divide x-coordinates by image width and y-coordinates\n    by image height.\n\n    Args:\n        bbox (tuple): Denormalized bounding box `(x_min, y_min, x_max, y_max)`.\n        rows (int): Image height.\n        cols (int): Image width.\n\n    Returns:\n        tuple: Normalized bounding box `(x_min, y_min, x_max, y_max)`.\n\n    Raises:\n        ValueError: If rows or cols is less or equal zero\n\n    \"\"\"\n    (x_min, y_min, x_max, y_max), tail = bbox[:4], tuple(bbox[4:])\n\n    if rows <= 0:\n        raise ValueError(\"Argument rows must be positive integer\")\n    if cols <= 0:\n        raise ValueError(\"Argument cols must be positive integer\")\n\n    x_min, x_max = x_min / cols, x_max / cols\n    y_min, y_max = y_min / rows, y_max / rows\n\n    return (x_min, y_min, x_max, y_max) + tail\n\n\ndef denormalize_bbox(bbox, rows, cols):\n    \"\"\"Denormalize coordinates of a bounding box. Multiply x-coordinates by image width and y-coordinates\n    by image height. This is an inverse operation for :func:`~albumentations.augmentations.bbox.normalize_bbox`.\n\n    Args:\n        bbox (tuple): Normalized bounding box `(x_min, y_min, x_max, y_max)`.\n        rows (int): Image height.\n        cols (int): Image width.\n\n    Returns:\n        tuple: Denormalized bounding box `(x_min, y_min, x_max, y_max)`.\n\n    Raises:\n        ValueError: If rows or cols is less or equal zero\n\n    \"\"\"\n    (x_min, y_min, x_max, y_max), tail = bbox[:4], tuple(bbox[4:])\n\n    if rows <= 0:\n        raise ValueError(\"Argument rows must be positive integer\")\n    if cols <= 0:\n        raise ValueError(\"Argument cols must be positive integer\")\n\n    x_min, x_max = x_min * cols, x_max * cols\n    y_min, y_max = y_min * rows, y_max * rows\n\n    return (x_min, y_min, x_max, y_max) + tail\n\n\ndef normalize_bboxes(bboxes, rows, cols):\n    \"\"\"Normalize a list of bounding boxes.\n\n    Args:\n        bboxes (List[tuple]): Denormalized bounding boxes `[(x_min, y_min, x_max, y_max)]`.\n        rows (int): Image height.\n        cols (int): Image width.\n\n    Returns:\n        List[tuple]: Normalized bounding boxes `[(x_min, y_min, x_max, y_max)]`.\n\n    \"\"\"\n    return [normalize_bbox(bbox, rows, cols) for bbox in bboxes]\n\n\ndef denormalize_bboxes(bboxes, rows, cols):\n    \"\"\"Denormalize a list of bounding boxes.\n\n    Args:\n        bboxes (List[tuple]): Normalized bounding boxes `[(x_min, y_min, x_max, y_max)]`.\n        rows (int): Image height.\n        cols (int): Image width.\n\n    Returns:\n        List[tuple]: Denormalized bounding boxes `[(x_min, y_min, x_max, y_max)]`.\n\n    \"\"\"\n    return [denormalize_bbox(bbox, rows, cols) for bbox in bboxes]\n\n\nclass DatasetRetriever(Dataset):\n    def __init__(self, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image, boxes, area, labels = self.load_image_and_boxes(index)       \n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            boxes = sample['bboxes']\n            labels = sample['labels']\n        \n        _,h,w = image.shape\n        boxes = normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        area = torch.as_tensor(boxes[:,2]*boxes[:,3], dtype=torch.float32)\n        labels =  np.zeros(len(boxes), dtype=np.int32)\n        return image, boxes, area, labels\n\n# Statistics\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n# model definition\nclass DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries, pretrained=False):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=pretrained)\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)\n\n# Fitter\nclass Fitter:\n    def __init__(self, model, criterion, device, config):\n        self.config = config\n        self.epoch = 0\n        self.criterion = criterion\n\n        self.base_dir = f'./{config.folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10 ** 9\n\n        self.train_his = pd.DataFrame(columns=['train_loss', 'val_loss'])\n\n        self.model = model\n        self.device = device\n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch(train_loader)\n            train_loss_avg = summary_loss.avg\n\n            self.log(\n                f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n            t = time.time()\n            summary_loss = self.validation(validation_loader)\n            val_loss_avg = summary_loss.avg\n            self.log(\n                f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.train_his = self.train_his.append(\n                {'epoch': self.epoch, 'train_loss': train_loss_avg, 'val_loss': val_loss_avg}, ignore_index=True)\n            self.train_his.to_csv('train_his.csv')\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        self.criterion.eval()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                images = list(image.to(self.device) for image in images)\n                batch_size = len(images)\n                targets = [{k: v.to(self.device) for k, v in t.items()} for t in targets]\n\n                output = self.model(images)\n\n                loss_dict = self.criterion(output, targets)\n                weight_dict = self.criterion.weight_dict\n\n                losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n                summary_loss.update(losses.detach().item(), batch_size)\n\n        return summary_loss\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        self.criterion.train()\n        summary_loss = AverageMeter()\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n\n            \n            images = list(image.to(self.device) for image in images)\n            batch_size = len(images)\n            targets = [{k: v.to(self.device) for k, v in t.items()} for t in targets]\n\n            self.optimizer.zero_grad()\n\n            output = self.model(images)\n\n            loss_dict = self.criterion(output, targets)\n            weight_dict = self.criterion.weight_dict\n\n            losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            losses.backward()\n\n            summary_loss.update(losses.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss\n\n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n\n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')\n\n\nclass TrainGlobalConfig:\n    folder = CACHE_DIR + '/detr'\n\n    num_workers = 2\n    batch_size = 8\n    n_epochs = 50\n    lr = 2e-5\n    num_classes = 2\n    num_queries = 100\n    null_class_coef = 0.5\n\n    \n    verbose = True\n    verbose_step = 1\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=1,\n        verbose=False,\n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0,\n        min_lr=1e-8,\n        eps=1e-08\n    )\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\n\ndef train_detr(marking, df_folds, fold):\n    # prepare dataset and dataloader\n    train_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] != fold].index.values,\n        marking=marking,\n        transforms=get_train_transforms(),\n        test=False,\n    )\n    validation_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] == fold].index.values,\n        marking=marking,\n        transforms=get_valid_transforms(),\n        test=True,\n    )\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        sampler=RandomSampler(train_dataset),\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n        collate_fn=collate_fn,\n    )\n    val_loader = DataLoader(\n        validation_dataset,\n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n        collate_fn=collate_fn,\n    )\n\n    matcher = HungarianMatcher()\n    weight_dict = weight_dict = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}\n    losses = ['labels', 'boxes', 'cardinality']\n\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n    model = DETRModel(num_classes=TrainGlobalConfig.num_classes,num_queries=TrainGlobalConfig.num_queries, pretrained=True)\n    model = model.to(device)\n\n    criterion = SetCriterion(TrainGlobalConfig.num_classes-1, matcher, weight_dict, eos_coef = TrainGlobalConfig.null_class_coef, losses=losses)\n    criterion = criterion.to(device)\n\n    fitter = Fitter(model=model, criterion=criterion, device=device, config=TrainGlobalConfig)\n    fitter.fit(train_loader, val_loader)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-24T04:28:12.080654Z","iopub.execute_input":"2023-05-24T04:28:12.081027Z","iopub.status.idle":"2023-05-24T04:28:12.161387Z","shell.execute_reply.started":"2023-05-24T04:28:12.080999Z","shell.execute_reply":"2023-05-24T04:28:12.160152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(SEED)\n# set fold for validation\nFOLD = 0\n# read dataset metadata\nmarking = pd.read_csv(DATA_DIR + '/train.csv')\n# convert string to int list\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n# reformat the target bounding boxes for yolo format, F-RCNN format...\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:, i]\nmarking.drop(columns=['bbox'], inplace=True)\nmarking['x_center'] = marking['x'] + marking['w']/2\nmarking['y_center'] = marking['y'] + marking['h']/2\n# single class in this dataset\nmarking['classes'] = 0\n\n# group data uniformly\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\ndf_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count() # calculate the target frequency for each image\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source'] # group by source\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\n# assign fold number\ndf_folds.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","metadata":{"execution":{"iopub.status.busy":"2023-05-24T04:28:12.163636Z","iopub.execute_input":"2023-05-24T04:28:12.164433Z","iopub.status.idle":"2023-05-24T04:28:13.442997Z","shell.execute_reply.started":"2023-05-24T04:28:12.164398Z","shell.execute_reply":"2023-05-24T04:28:13.441770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_detr(marking, df_folds, FOLD)","metadata":{"execution":{"iopub.status.busy":"2023-05-24T04:28:13.445822Z","iopub.execute_input":"2023-05-24T04:28:13.446615Z","iopub.status.idle":"2023-05-24T04:39:51.188887Z","shell.execute_reply.started":"2023-05-24T04:28:13.446578Z","shell.execute_reply":"2023-05-24T04:39:51.187258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-05-24T04:39:51.190603Z","iopub.status.idle":"2023-05-24T04:39:51.191309Z","shell.execute_reply.started":"2023-05-24T04:39:51.191063Z","shell.execute_reply":"2023-05-24T04:39:51.191090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}