{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import DETR of Facebook","metadata":{}},{"cell_type":"code","source":"# Download detr repos\n!git clone https://github.com/facebookresearch/detr.git ","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-03T03:14:31.342443Z","iopub.execute_input":"2022-03-03T03:14:31.343530Z","iopub.status.idle":"2022-03-03T03:14:33.112000Z","shell.execute_reply.started":"2022-03-03T03:14:31.343411Z","shell.execute_reply":"2022-03-03T03:14:33.111190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install utils\n!pip install glob2\n!pip install --upgrade pip","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:33.115421Z","iopub.execute_input":"2022-03-03T03:14:33.115696Z","iopub.status.idle":"2022-03-03T03:14:55.075407Z","shell.execute_reply.started":"2022-03-03T03:14:33.115668Z","shell.execute_reply":"2022-03-03T03:14:55.074564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic\nimport os\nimport numpy as np \nimport pandas as pd \nfrom tqdm.autonotebook import tqdm\n\n#computer vison module\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom PIL import Image\n\n#pytorch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n#for K-fold cross validation\nfrom sklearn.model_selection import StratifiedKFold\n\n#Hungarian loss & bipartite matcher\nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n\n#Data aug\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob2 import glob","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-03-03T03:14:55.077992Z","iopub.execute_input":"2022-03-03T03:14:55.078418Z","iopub.status.idle":"2022-03-03T03:14:58.289124Z","shell.execute_reply.started":"2022-03-03T03:14:55.078377Z","shell.execute_reply":"2022-03-03T03:14:58.288338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data pre-processing","metadata":{}},{"cell_type":"code","source":"train_dir = '../input/global-wheat-detection/train/'\ntest_dir = '../input/global-wheat-detection/test/'\n\ntrain_annos = '../input/global-wheat-detection/train.csv'\n\n# glob to get lists of files of each dir \ntrain_fns = glob(train_dir + '*')\ntest_fns = glob(test_dir + '*')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:58.290485Z","iopub.execute_input":"2022-03-03T03:14:58.290822Z","iopub.status.idle":"2022-03-03T03:14:58.612779Z","shell.execute_reply.started":"2022-03-03T03:14:58.290788Z","shell.execute_reply":"2022-03-03T03:14:58.612107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of train images: ', len(train_fns))\nprint('Number of test images: ', len(test_fns))","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:58.616763Z","iopub.execute_input":"2022-03-03T03:14:58.617028Z","iopub.status.idle":"2022-03-03T03:14:58.622035Z","shell.execute_reply.started":"2022-03-03T03:14:58.617003Z","shell.execute_reply":"2022-03-03T03:14:58.621002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(train_annos)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:58.625297Z","iopub.execute_input":"2022-03-03T03:14:58.625915Z","iopub.status.idle":"2022-03-03T03:14:58.876745Z","shell.execute_reply.started":"2022-03-03T03:14:58.625876Z","shell.execute_reply":"2022-03-03T03:14:58.875956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create dataframe with all training images\nall_train_images = pd.DataFrame([fns.split('/')[-1][:-4] for fns in train_fns])\nall_train_images.columns=['image_id']\n\n#Merge with bboxes\nall_train_images = all_train_images.merge(train, on='image_id', how='left')\n\n#Fill nan values with zero\nall_train_images['bbox'] = all_train_images.bbox.fillna('[0,0,0,0]')\n\n#Split 4 values of bbox to columns\nbbox_items = all_train_images.bbox.str.split(',', expand = True)\n\nall_train_images['x_min'] = bbox_items[0].str.strip('[ ').astype(float)\nall_train_images['y_min'] = bbox_items[1].str.strip(' ').astype(float)\nall_train_images['width'] = bbox_items[2].str.strip(' ').astype(float)\nall_train_images['height'] = bbox_items[3].str.strip('] ').astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:58.878197Z","iopub.execute_input":"2022-03-03T03:14:58.878514Z","iopub.status.idle":"2022-03-03T03:14:59.824817Z","shell.execute_reply.started":"2022-03-03T03:14:58.878480Z","shell.execute_reply":"2022-03-03T03:14:59.824010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check imgs without bboxes.\nprint('There are {} images with no bounding box!'.format(len(all_train_images)- len(train)))","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:59.826087Z","iopub.execute_input":"2022-03-03T03:14:59.826422Z","iopub.status.idle":"2022-03-03T03:14:59.833649Z","shell.execute_reply.started":"2022-03-03T03:14:59.826388Z","shell.execute_reply":"2022-03-03T03:14:59.832618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove img without bboxes.\nall_train_images = all_train_images[all_train_images.width != 0]\n\nprint('There are {} images with no bounding box!'.format(len(all_train_images)- len(train)))","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:59.835117Z","iopub.execute_input":"2022-03-03T03:14:59.835674Z","iopub.status.idle":"2022-03-03T03:14:59.870264Z","shell.execute_reply.started":"2022-03-03T03:14:59.835634Z","shell.execute_reply":"2022-03-03T03:14:59.869362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This code is used for connecting boxes with id, we partly get from UIT LOGO baseline. \ndef get_all_bboxes(df, image_id):\n    image_bboxes = df[df.image_id == image_id]   \n    bboxes = []\n    for _,row in image_bboxes.iterrows():\n        bboxes.append((row.x_min, row.y_min, row.width, row.height))\n        \n    return bboxes\n\ndef plot_image_examples(df, rows=3, cols=3):\n    fig, axs = plt.subplots(rows, cols, figsize=(25,20))\n    for row in range(rows):\n        for col in range(cols):\n            idx = np.random.randint(len(df), size=1)[0]\n            img_id = df.iloc[idx].image_id\n            \n            img = Image.open(train_dir + img_id + '.jpg')\n            axs[row, col].imshow(img)\n            \n            bboxes = get_all_bboxes(df, img_id)\n            \n            for bbox in bboxes:\n                rect = patches.Rectangle((bbox[0],bbox[1]),bbox[2],bbox[3],linewidth=1,edgecolor='r',facecolor='none')\n                axs[row, col].add_patch(rect)\n            \n            axs[row, col].axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:59.871538Z","iopub.execute_input":"2022-03-03T03:14:59.872035Z","iopub.status.idle":"2022-03-03T03:14:59.881249Z","shell.execute_reply.started":"2022-03-03T03:14:59.871998Z","shell.execute_reply":"2022-03-03T03:14:59.880443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample images with bounding box.\nplot_image_examples(all_train_images)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:14:59.882582Z","iopub.execute_input":"2022-03-03T03:14:59.883001Z","iopub.status.idle":"2022-03-03T03:15:03.308381Z","shell.execute_reply.started":"2022-03-03T03:14:59.882963Z","shell.execute_reply":"2022-03-03T03:15:03.306957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop old bbox column\nall_train_images.drop(columns = ['bbox'], inplace=True)\n\n# Count bbox with condition\nall_train_images['count'] = all_train_images.apply(lambda row: 1 if np.isfinite(row.width) else 0, axis=1)\n\nall_train_images","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:03.309694Z","iopub.execute_input":"2022-03-03T03:15:03.310060Z","iopub.status.idle":"2022-03-03T03:15:05.874784Z","shell.execute_reply.started":"2022-03-03T03:15:03.310015Z","shell.execute_reply":"2022-03-03T03:15:05.873840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Fold","metadata":{}},{"cell_type":"code","source":"# Config\nn_folds = 4\nseed = 42\nnum_classes = 2 #(wheat and no-obj class)\nnum_queries = 100\nnull_class_coef = 0.5\nBATCH_SIZE = 12\nLR = 2e-5\nEPOCHS = 20","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:05.876139Z","iopub.execute_input":"2022-03-03T03:15:05.876491Z","iopub.status.idle":"2022-03-03T03:15:05.881785Z","shell.execute_reply.started":"2022-03-03T03:15:05.876452Z","shell.execute_reply":"2022-03-03T03:15:05.880808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"[1]\"\"\"\n\nskf = StratifiedKFold(n_splits= n_folds, shuffle=True, random_state=42)\n\ndf_folds = all_train_images[['image_id', 'count']].copy()\n\n#Gather all boxes of each img \ndf_folds = df_folds.groupby('image_id').count()\n\n#Get source for each img, we stratify to 4 folds.\ndf_folds.loc[:, 'source'] = all_train_images[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n    )\n\ndf_folds.loc[:, 'fold'] = 0\n\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:05.883324Z","iopub.execute_input":"2022-03-03T03:15:05.883906Z","iopub.status.idle":"2022-03-03T03:15:06.313310Z","shell.execute_reply.started":"2022-03-03T03:15:05.883860Z","shell.execute_reply":"2022-03-03T03:15:06.312383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_folds","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.314558Z","iopub.execute_input":"2022-03-03T03:15:06.315079Z","iopub.status.idle":"2022-03-03T03:15:06.329378Z","shell.execute_reply.started":"2022-03-03T03:15:06.315041Z","shell.execute_reply":"2022-03-03T03:15:06.328422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create dataset object","metadata":{}},{"cell_type":"code","source":"\"\"\"[2]\"\"\"\nclass WheatDataset(Dataset):\n    \n    def __init__(self,image_ids,dataframe,transforms=None):\n        self.image_ids = image_ids\n        self.df = dataframe        \n        \n        #Data augmentation \n        self.transforms = transforms\n        \n        \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n    \n    def __getitem__(self,index):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n        \n        image = cv2.imread(f'{train_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        # DETR takes in data in coco format \n        boxes = records[['x_min', 'y_min', 'width', 'height']].values\n        \n        #Area of bb\n        area = boxes[:,2]*boxes[:,3]\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        labels =  np.zeros(len(boxes), dtype=np.int32)\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            boxes = sample['bboxes']\n            labels = sample['labels']            \n            \n        #Normalizing BBOXES            \n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        \n        return image, target, image_id","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.330723Z","iopub.execute_input":"2022-03-03T03:15:06.331131Z","iopub.status.idle":"2022-03-03T03:15:06.344074Z","shell.execute_reply.started":"2022-03-03T03:15:06.331096Z","shell.execute_reply":"2022-03-03T03:15:06.343054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create DETR object","metadata":{}},{"cell_type":"code","source":"\"\"\"[3]\"\"\"\nclass DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        #Download pretrained DETR model with backbone resnet101 feature extractor.\n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet101', pretrained=True)\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.345560Z","iopub.execute_input":"2022-03-03T03:15:06.345999Z","iopub.status.idle":"2022-03-03T03:15:06.354448Z","shell.execute_reply.started":"2022-03-03T03:15:06.345961Z","shell.execute_reply":"2022-03-03T03:15:06.353685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pretrained DETR with frozen transformer.\nclass frozen_transformer_DETR(nn.Module):\n    def __init__(self,num_classes, num_queries):\n        super(frozen_transformer_DETR, self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        #Download pretrained DETR model with backbone resnet101 feature extractor.\n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet101', pretrained=True)\n        \n        \n        self.model.transformer.encoder.layers.requires_grad = False\n        self.model.transformer.decoder.layers.requires_grad = False\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.356040Z","iopub.execute_input":"2022-03-03T03:15:06.356389Z","iopub.status.idle":"2022-03-03T03:15:06.365928Z","shell.execute_reply.started":"2022-03-03T03:15:06.356351Z","shell.execute_reply":"2022-03-03T03:15:06.365026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create matcher for loss calculation and set \"lambda\" params\n# loss_ce: classification loss (Hungarian matching loss)\n# loss_bbox: bbox loss\n# giou: giou loss\n\nmatcher = HungarianMatcher()\n\nweight_dict = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}\n\nlosses = ['labels', 'boxes', 'cardinality']","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.367436Z","iopub.execute_input":"2022-03-03T03:15:06.368006Z","iopub.status.idle":"2022-03-03T03:15:06.375020Z","shell.execute_reply.started":"2022-03-03T03:15:06.367925Z","shell.execute_reply":"2022-03-03T03:15:06.374204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"\"\"\"[4]\"\"\"\ndef get_train_transforms():\n    return A.Compose([A.OneOf([A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),                               \n                      A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.9)],p=0.9),                      \n                      A.ToGray(p=0.01),                      \n                      A.HorizontalFlip(p=0.5),                      \n                      A.VerticalFlip(p=0.5),                      \n                      A.Resize(height=512, width=512, p=1),                      \n                      A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),                      \n                      ToTensorV2(p=1.0)],                      \n                      p=1.0,                     \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.376692Z","iopub.execute_input":"2022-03-03T03:15:06.377059Z","iopub.status.idle":"2022-03-03T03:15:06.386861Z","shell.execute_reply.started":"2022-03-03T03:15:06.377024Z","shell.execute_reply":"2022-03-03T03:15:06.385907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss calculation object","metadata":{}},{"cell_type":"code","source":"# To compute and store avg and current value.\nclass AverageMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.388194Z","iopub.execute_input":"2022-03-03T03:15:06.388585Z","iopub.status.idle":"2022-03-03T03:15:06.397561Z","shell.execute_reply.started":"2022-03-03T03:15:06.388548Z","shell.execute_reply":"2022-03-03T03:15:06.396775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    \n    model.train()\n    criterion.train()\n    \n    summary_loss = AverageMeter()\n    \n    with tqdm(data_loader, total=len(data_loader)) as tk0:\n    \n        for step, (images, targets, image_ids) in enumerate(tk0):\n        \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n        \n            losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        \n            optimizer.zero_grad()\n\n            losses.backward()\n            optimizer.step()\n            \n            if scheduler is not None:\n                scheduler.step()\n        \n            summary_loss.update(losses.item(),BATCH_SIZE)\n            tk0.set_postfix(loss=summary_loss.avg)\n        \n    return summary_loss","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.399294Z","iopub.execute_input":"2022-03-03T03:15:06.399741Z","iopub.status.idle":"2022-03-03T03:15:06.409780Z","shell.execute_reply.started":"2022-03-03T03:15:06.399703Z","shell.execute_reply":"2022-03-03T03:15:06.408828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    \n    model.eval()\n    criterion.eval()\n    \n    summary_loss = AverageMeter()\n    \n    # Disabled gradient calculation over the weights\n    with torch.no_grad():\n        \n        tk0 = tqdm(data_loader, total=len(data_loader))\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n        \n            losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            \n            summary_loss.update(losses.item(),BATCH_SIZE)\n            tk0.set_postfix(loss=summary_loss.avg)\n    \n    return summary_loss","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.410997Z","iopub.execute_input":"2022-03-03T03:15:06.411406Z","iopub.status.idle":"2022-03-03T03:15:06.420999Z","shell.execute_reply.started":"2022-03-03T03:15:06.411370Z","shell.execute_reply":"2022-03-03T03:15:06.419791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.422504Z","iopub.execute_input":"2022-03-03T03:15:06.423061Z","iopub.status.idle":"2022-03-03T03:15:06.429955Z","shell.execute_reply.started":"2022-03-03T03:15:06.423025Z","shell.execute_reply":"2022-03-03T03:15:06.429120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"[5]\"\"\"\ndef get_data_loader(fold):    \n    df_train = df_folds[df_folds['fold'] != fold]\n    df_valid = df_folds[df_folds['fold'] == fold]\n\n    train_dataset = WheatDataset(\n        image_ids=df_train.index.values,\n        dataframe=all_train_images,\n        transforms=get_train_transforms()\n        )\n\n    train_data_loader = DataLoader(\n        train_dataset,\n        batch_size=BATCH_SIZE,\n        shuffle=False,\n        num_workers=4,\n        collate_fn=collate_fn\n        )\n\n    valid_dataset = WheatDataset(\n        image_ids=df_valid.index.values,\n        dataframe=all_train_images,\n        transforms=get_valid_transforms()\n        )\n\n    valid_data_loader = DataLoader(\n        valid_dataset,\n        batch_size=BATCH_SIZE,\n        shuffle=False,\n        num_workers=4,\n        collate_fn=collate_fn\n        )\n    \n    return train_data_loader, valid_data_loader","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.431534Z","iopub.execute_input":"2022-03-03T03:15:06.431956Z","iopub.status.idle":"2022-03-03T03:15:06.439822Z","shell.execute_reply.started":"2022-03-03T03:15:06.431922Z","shell.execute_reply":"2022-03-03T03:15:06.438870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trainnnn(train_data_loader, valid_data_loader, device, model, criterion, optimizer, fold, best_loss, index):\n    for epoch in range(EPOCHS):\n        train_loss = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        valid_loss = eval_fn(valid_data_loader, model,criterion, device)\n        \n        print('|EPOCH {}| TRAIN_LOSS {}| VALID_LOSS {}|'.format(epoch+1,train_loss.avg,valid_loss.avg))\n        \n        if valid_loss.avg < best_loss:\n            best_loss = valid_loss.avg\n            print('Best model found for Fold {} in Epoch {}........Saving Model'.format(fold,epoch+1))\n            torch.save(model.state_dict(), f'THE_BEST_{index}_fold_{fold}.pth')\n        \n        index+=1","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.442285Z","iopub.execute_input":"2022-03-03T03:15:06.442522Z","iopub.status.idle":"2022-03-03T03:15:06.450073Z","shell.execute_reply.started":"2022-03-03T03:15:06.442500Z","shell.execute_reply":"2022-03-03T03:15:06.449128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"For training the whole Pretrained DETR\"\"\"\n#Set device, load model and set criteron\ndevice = torch.device('cuda')\n\nmodel = DETRModel(num_classes=num_classes, num_queries=num_queries)\nmodel = model.to(device)\n\ncriterion = SetCriterion(num_classes-1, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\ncriterion = criterion.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:06.451459Z","iopub.execute_input":"2022-03-03T03:15:06.452136Z","iopub.status.idle":"2022-03-03T03:15:24.598271Z","shell.execute_reply.started":"2022-03-03T03:15:06.452099Z","shell.execute_reply":"2022-03-03T03:15:24.597327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"[OR] For training Pretrained DETR with frozen transformer\"\"\"\n\"\"\"\n#Set device, load model and set criteron\ndevice = torch.device('cuda')\n\nmodel = frozen_transformer_DETR(num_classes=num_classes, num_queries=num_queries)\nprint('Total parameters of DETR: ', sum(p.numel() for p in model.parameters()))\nprint('We now fine-tune Transformer by freezing the pretrained model!')\nprint('Total of params of MLP layers for clases and boxes.', sum(p.numel() for p in model.parameters() if p.requires_grad))\nmodel = model.to(device)\n\ncriterion = SetCriterion(num_classes-1, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\ncriterion = criterion.to(device)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:24.599761Z","iopub.execute_input":"2022-03-03T03:15:24.600137Z","iopub.status.idle":"2022-03-03T03:15:24.605757Z","shell.execute_reply.started":"2022-03-03T03:15:24.600099Z","shell.execute_reply":"2022-03-03T03:15:24.604955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_loss = 10**5\nindex = 0\n    \n#AdamW optimizer\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n\n#Train 4 folds, each fold 10 epochs.\nfor fold in range(n_folds):\n    train_data_loader, valid_data_loader = get_data_loader(fold=fold)\n    trainnnn(train_data_loader, valid_data_loader, device, model, criterion, optimizer, fold, best_loss, index)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:24.607032Z","iopub.execute_input":"2022-03-03T03:15:24.607630Z","iopub.status.idle":"2022-03-03T03:15:34.334630Z","shell.execute_reply.started":"2022-03-03T03:15:24.607593Z","shell.execute_reply":"2022-03-03T03:15:34.331786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Recall and test model ","metadata":{}},{"cell_type":"code","source":"#Recall model\ndevice = torch.device(\"cuda\")\n\nmodel = frozen_transformer_DETR(num_classes = num_classes, num_queries=num_queries)\n# model.load_state_dict(torch.load('./THE_BEST_9_fold_3.pth'))\nmodel.load_state_dict(torch.load('../input/v14unfreezen/THE_BEST_0_fold_3.pth'))\nmodel = model.to(device)\n\ncriterion = SetCriterion(num_classes-1, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\ncriterion = criterion.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:45.429144Z","iopub.execute_input":"2022-03-03T03:15:45.429477Z","iopub.status.idle":"2022-03-03T03:15:50.880645Z","shell.execute_reply.started":"2022-03-03T03:15:45.429447Z","shell.execute_reply":"2022-03-03T03:15:50.879593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create test dataset object\nTo parse annotation of test images","metadata":{}},{"cell_type":"code","source":"class WheatTestDataset(Dataset):\n    \n    def __init__(self, image_dir, dataframe, transforms=None):\n        super().__init__()\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n    \n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n\n        image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        boxes = records[['xmin', 'ymin', 'width', 'height']].values\n\n        if self.transforms:\n            sample = {\n                'image': image,\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        return image, image_id, boxes\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:00.129377Z","iopub.execute_input":"2022-03-03T03:16:00.129714Z","iopub.status.idle":"2022-03-03T03:16:00.138212Z","shell.execute_reply.started":"2022-03-03T03:16:00.129684Z","shell.execute_reply":"2022-03-03T03:16:00.137191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_transforms():\n    return A.Compose([\n        A.Resize(512, 512),\n        ToTensorV2(p=1.0)\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:03.332506Z","iopub.execute_input":"2022-03-03T03:16:03.332827Z","iopub.status.idle":"2022-03-03T03:16:03.338998Z","shell.execute_reply.started":"2022-03-03T03:16:03.332797Z","shell.execute_reply":"2022-03-03T03:16:03.337055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test annotation processing","metadata":{}},{"cell_type":"code","source":"test_annos = pd.read_csv('../input/d/zhangshh/test-annotations/_annotations.csv')\ntest_annos","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:05.388512Z","iopub.execute_input":"2022-03-03T03:16:05.388889Z","iopub.status.idle":"2022-03-03T03:16:05.422095Z","shell.execute_reply.started":"2022-03-03T03:16:05.388833Z","shell.execute_reply":"2022-03-03T03:16:05.421403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq = '_jpg.rf.'\ntest_annos['filename'] = [name.split(seq, 1)[0] for name in test_annos['filename']]\n\ntest_annos = test_annos.rename(columns = {'filename': 'image_id'})\n\ntest_annos.drop(columns = ['class'], inplace=True)\ntest_annos.drop(columns = ['width'], inplace=True)\ntest_annos.drop(columns = ['height'], inplace=True)\n\ntest_annos['width'] = test_annos['xmax'] - test_annos['xmin']\ntest_annos['height'] = test_annos['ymax'] - test_annos['ymin']\n\ntest_annos.drop(columns = ['xmax'], inplace=True)\ntest_annos.drop(columns = ['ymax'], inplace=True)\n\ntest_annos","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:07.663315Z","iopub.execute_input":"2022-03-03T03:16:07.663650Z","iopub.status.idle":"2022-03-03T03:16:07.690564Z","shell.execute_reply.started":"2022-03-03T03:16:07.663618Z","shell.execute_reply":"2022-03-03T03:16:07.689788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get test dataframe \ntest_df = test_annos\n\ntest_dataset = WheatTestDataset(image_dir = test_dir,\n                                dataframe = test_df,  \n                                transforms = get_test_transforms())\n\ntest_data_loader = DataLoader(test_dataset,\n                              batch_size=10,\n                              shuffle=False,\n                              num_workers=4,\n                              drop_last=False,\n                              collate_fn=collate_fn\n                             )","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:10.682162Z","iopub.execute_input":"2022-03-03T03:16:10.682470Z","iopub.status.idle":"2022-03-03T03:16:10.689400Z","shell.execute_reply.started":"2022-03-03T03:16:10.682441Z","shell.execute_reply":"2022-03-03T03:16:10.688233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"[6]\"\"\"\nconfidence_threshold = 0.6\n\ndef predict_and_view(test_data_loader, model, confidence_threshold): \n    \n    images, image_ids, all_test_boxes = next(iter(test_data_loader))\n    \n    _, h, w = images[0].shape # for DE norm boxes\n    \n    images = list(image.to(device) for image in images)\n    \n    all_pred_boxes = []\n    \n    # Predict \n    with torch.no_grad():\n        outputs = model(images)   \n    \n    outputs = [{k: v.to(device) for k, v in outputs.items()}]    \n        \n    for i, image in enumerate(images):\n        \n        # For plotting test annotation boxes. \n        test_boxes = all_test_boxes[i]\n        \n        sample = image.permute(1,2,0).cpu().numpy()\n        \n        fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n        \n        for box in test_boxes:\n            color = (220, 0, 0)\n            cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2]+box[0], box[3]+box[1]),\n                  color, 1)\n            # print(box[0], box[1], box[2],box[3])\n        \n        # For plotting predicted boxes.\n        pred_boxes = outputs[0]['pred_boxes'][i].detach().cpu().numpy()\n        pred_boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(pred_boxes, h, w)] # DE norm\n        prob  = outputs[0]['pred_logits'][i].softmax(1).detach().cpu().numpy()[:,0]\n    \n        \n        list_pred_boxes = np.empty(shape=[4, 1])\n        \n        for box, p in zip(pred_boxes, prob):        \n            if p > confidence_threshold:\n                color = (0,0,220) \n                cv2.rectangle(sample,\n                      (box[0], box[1]),\n                      (box[2]+box[0], box[3]+box[1]),\n                      color, 1)\n                \n                # For eval \n                box[2] = box[2]+box[0]\n                box[3] = box[3]+box[1]\n                \n                box = box.reshape(4,1)\n                list_pred_boxes = np.concatenate((list_pred_boxes,box),axis=1)\n                \n        all_pred_boxes.append(list_pred_boxes)\n        \n        ax.set_axis_off()\n        ax.imshow(sample)\n        \n    return all_pred_boxes, all_test_boxes","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:13.019160Z","iopub.execute_input":"2022-03-03T03:16:13.019488Z","iopub.status.idle":"2022-03-03T03:16:13.035926Z","shell.execute_reply.started":"2022-03-03T03:16:13.019460Z","shell.execute_reply":"2022-03-03T03:16:13.033972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np_all_pred_boxes, np_all_test_boxes = predict_and_view(test_data_loader, model, confidence_threshold=confidence_threshold)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:16.806842Z","iopub.execute_input":"2022-03-03T03:16:16.807192Z","iopub.status.idle":"2022-03-03T03:16:19.887236Z","shell.execute_reply.started":"2022-03-03T03:16:16.807162Z","shell.execute_reply":"2022-03-03T03:16:19.886388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"code","source":"all_pred_boxes = []\nall_test_boxes = []\n\nfor pred_boxes in np_all_pred_boxes:\n    # Transpose the matrix of boxes\n    pred_boxes = pred_boxes.transpose()\n    # Convert to torch tensor\n    pred_boxes = torch.from_numpy(pred_boxes)\n    all_pred_boxes.append(pred_boxes)\n    \nfor img_test_boxes in np_all_test_boxes:\n    \n    for box in img_test_boxes:\n        box[2] = box[2]+box[0]\n        box[3] = box[3]+box[1]\n        \n    # Convert to torch tensor\n    img_test_boxes = torch.from_numpy(img_test_boxes)\n    all_test_boxes.append(img_test_boxes)  ","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:16:57.934447Z","iopub.execute_input":"2022-03-03T03:16:57.934800Z","iopub.status.idle":"2022-03-03T03:16:57.941665Z","shell.execute_reply.started":"2022-03-03T03:16:57.934768Z","shell.execute_reply":"2022-03-03T03:16:57.940834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10):\n    print(all_pred_boxes[i].shape, ' -- ', all_test_boxes[i].shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:17:00.977717Z","iopub.execute_input":"2022-03-03T03:17:00.978073Z","iopub.status.idle":"2022-03-03T03:17:00.984073Z","shell.execute_reply.started":"2022-03-03T03:17:00.978042Z","shell.execute_reply":"2022-03-03T03:17:00.983003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_intersection(box, test_boxes):\n    \"\"\"Find the intersection of every box combination between two sets of boxes that are in boundary coordinates.\n    \"\"\"\n    # PyTorch auto-broadcasts singleton dimensions\n    lower_bounds = torch.max(box[:, :2].unsqueeze(1), test_boxes[:, :2].unsqueeze(0))  \n    upper_bounds = torch.min(box[:, 2:].unsqueeze(1), test_boxes[:, 2:].unsqueeze(0))  \n    intersection_dims = torch.clamp(upper_bounds - lower_bounds, min=0)  \n    \n    return intersection_dims[:, :, 0] * intersection_dims[:, :, 1]  \n\n\ndef find_iou(box, test_boxes):\n    \"\"\"Find IoU overlap of every box combination between two sets of boxes that are in boundary coordinates.\n    \"\"\"        \n    # Find intersections\n    intersection = find_intersection(box, test_boxes)  \n\n    # Find areas of each box in both sets\n    areas_box = (box[:, 2] - box[:, 0]) * (box[:, 3] - box[:, 1])  \n    areas_test_boxes = (test_boxes[:, 2] - test_boxes[:, 0]) * (test_boxes[:, 3] - test_boxes[:, 1]) \n\n    # Find the union\n    union = areas_box.unsqueeze(1) + areas_test_boxes.unsqueeze(0) - intersection  \n\n    return intersection / union ","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:17:03.751911Z","iopub.execute_input":"2022-03-03T03:17:03.752265Z","iopub.status.idle":"2022-03-03T03:17:03.761395Z","shell.execute_reply.started":"2022-03-03T03:17:03.752233Z","shell.execute_reply":"2022-03-03T03:17:03.759544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def AP(test_boxes, pred_boxes):\n    \"\"\"Calculate AP score of the prediction.\n    \"\"\"\n    average_precisions = torch.zeros((0), dtype=torch.float)\n    # All considered boxes\n    consi_boxes = 0\n    if test_boxes.size(0) > pred_boxes.size(0):\n        consi_boxes = test_boxes.size(0)\n    else:\n        consi_boxes = pred_boxes.size(0)\n        \n    true_pos = torch.zeros((consi_boxes), dtype=torch.float).to(device) \n    false_pos = torch.zeros((consi_boxes), dtype=torch.float).to(device)  \n    \n\n    for box in test_boxes:\n        this_box = box.unsqueeze(0)\n        overlaps = find_iou(this_box, pred_boxes)\n        max_overlap, idx = torch.max(overlaps.squeeze(0), dim=0)\n        \n        # Overlap < 0.3 is not acceptable \n        if max_overlap.item() < 0.3:\n            continue\n        \n        # Need it to pop detected box\n        og_id = torch.LongTensor(range(pred_boxes.size(0)))[idx]\n        pred_boxes = torch.cat([pred_boxes[0:og_id], pred_boxes[og_id+1:]])\n        \n        if max_overlap.item() >= 0.5:\n            true_pos[idx] = 1\n            \n        else:\n            false_pos[idx] = 1\n        \n        if pred_boxes.size(0) == 0:\n            break\n        \n    \n    sum_true_pos = torch.cumsum(true_pos, dim=0)\n    sum_false_pos = torch.cumsum(false_pos, dim=0)\n    \n    \n    # High precision means the accuracy of the bboxes found is high. \n    arr_precision = sum_true_pos / (\n                sum_true_pos + sum_false_pos + 1e-10)\n    # High recall means the rate of omitting the ground truth positive bboxes is low.\n    arr_recall = sum_true_pos/consi_boxes\n    \n    \n    # Order Precison and Recall like the ROC curve\n    # Find the mean of the maximum of the precisions corresponding to recalls above the threshold 't'\n    recall_thresholds = torch.arange(start=0, end=1.1, step=.1).tolist() # 0.0 ---> 1.0\n    precisions = torch.zeros((len(recall_thresholds)), dtype=torch.float).to(device)\n    \n    for i, t in enumerate(recall_thresholds):\n        recalls_above_t = arr_recall >= t\n        if recalls_above_t.any():\n            precisions[i] = arr_precision[recalls_above_t].max()\n        else:\n            precisions[i] = 0.0\n            \n        average_precision = precisions.mean().item()\n            \n        \n        \n    return arr_precision, arr_recall, average_precision","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:17:08.713465Z","iopub.execute_input":"2022-03-03T03:17:08.713794Z","iopub.status.idle":"2022-03-03T03:17:08.727752Z","shell.execute_reply.started":"2022-03-03T03:17:08.713763Z","shell.execute_reply":"2022-03-03T03:17:08.726796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display results and find mAP\nAPs = 0\nmAP = 0\n\n\nfor i in range(10):    \n    # precision = torch.zeros((), dtype=torch.float)\n    # recall = torch.zeros((24), dtype=torch.float)\n    \n    precision, recall, average_precision = AP(all_pred_boxes[i], all_test_boxes[i])\n    num_predict, num_test = all_pred_boxes[i].size(0), all_test_boxes[i].size(0)\n    \n    APs += average_precision\n    mAP = APs/10\n    \n    print('Image {}:'.format(i+1), '\\n', 'Number of Predicted boxes:', num_predict, end='\\n')\n    print(' Number of Ground truth boxes:', num_test, end='\\n')\n    \n    # print('Precision score: ',precision, end = '\\n')       \n    # print('Rcall score: ', recall, end = '\\n')\n    \n    \n    print('AP score: ',average_precision, end = '\\n')\n    print('------------------------------------', end ='\\n')\n    \nprint('mAP score: ',mAP, end = '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:17:11.941965Z","iopub.execute_input":"2022-03-03T03:17:11.942315Z","iopub.status.idle":"2022-03-03T03:17:12.071814Z","shell.execute_reply.started":"2022-03-03T03:17:11.942285Z","shell.execute_reply":"2022-03-03T03:17:12.071087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T03:15:34.360759Z","iopub.status.idle":"2022-03-03T03:15:34.361436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n\n[1] https://www.kaggle.com/shonenkov/training-efficientdet\n\n[2],\n[3],\n[4],\n[5]\nhttps://www.kaggle.com/tanulsingh077/end-to-end-object-detection-with-transformers-detr\n\n[6] https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train\n","metadata":{}}]}