{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!git clone https://github.com/facebookresearch/detr.git  /tmp/packages/detr #cloning github repo of detr to import its unique loss","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-20T09:39:29.827482Z","iopub.execute_input":"2022-03-20T09:39:29.827778Z","iopub.status.idle":"2022-03-20T09:39:32.893116Z","shell.execute_reply.started":"2022-03-20T09:39:29.827747Z","shell.execute_reply":"2022-03-20T09:39:32.891605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls /tmp/packages/detr/*","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:32.897437Z","iopub.execute_input":"2022-03-20T09:39:32.897869Z","iopub.status.idle":"2022-03-20T09:39:33.733146Z","shell.execute_reply.started":"2022-03-20T09:39:32.897821Z","shell.execute_reply":"2022-03-20T09:39:33.731887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Now if you have seen the video , you know that DETR uses a special loss called Bipartite Matching loss where it assigns one ground truth bbox to a predicted box using a matcher , thus when fine tuning we need the matcher (hungarian matcher as used in paper) and also the fucntion SetCriterion which gives Bipartite matching loss for backpropogation. This is the reason for forking the github repo","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm.auto import tqdm\n\n\n#Torch\nimport  \nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n#sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\n#CV\nimport cv2\n\n################# DETR FUCNTIONS FOR LOSS######################## \nimport sys\nsys.path.extend(['/tmp/packages/detr/'])\n\nfrom models.matcher import HungarianMatcher\nfrom models.detr import SetCriterion\n#################################################################\n\n#Albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob import glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-20T09:39:33.737518Z","iopub.execute_input":"2022-03-20T09:39:33.737842Z","iopub.status.idle":"2022-03-20T09:39:37.025779Z","shell.execute_reply.started":"2022-03-20T09:39:33.737808Z","shell.execute_reply":"2022-03-20T09:39:37.024796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        \n    @property\n    def avg(self):\n        return (self.sum / self.count) if self.count>0 else 0","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:37.027386Z","iopub.execute_input":"2022-03-20T09:39:37.027849Z","iopub.status.idle":"2022-03-20T09:39:37.038171Z","shell.execute_reply.started":"2022-03-20T09:39:37.02779Z","shell.execute_reply":"2022-03-20T09:39:37.036997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_folds = 5\nseed = 42\nnull_class_coef = 0.5\nnum_classes = 1\nnum_queries = 100\nBATCH_SIZE = 8\nLR = 5e-5\nlr_dict = {'backbone':0.1,'transformer':1,'embed':1,'final': 5}\nEPOCHS = 2\nmax_norm = 0\nmodel_name = 'detr_resnet50'","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:37.04391Z","iopub.execute_input":"2022-03-20T09:39:37.044919Z","iopub.status.idle":"2022-03-20T09:39:37.05384Z","shell.execute_reply.started":"2022-03-20T09:39:37.044872Z","shell.execute_reply":"2022-03-20T09:39:37.052442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed Everything\n\nSeeding everything for reproducible results","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:37.058844Z","iopub.execute_input":"2022-03-20T09:39:37.059416Z","iopub.status.idle":"2022-03-20T09:39:37.067867Z","shell.execute_reply.started":"2022-03-20T09:39:37.059308Z","shell.execute_reply":"2022-03-20T09:39:37.06657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:37.069783Z","iopub.execute_input":"2022-03-20T09:39:37.070682Z","iopub.status.idle":"2022-03-20T09:39:37.080842Z","shell.execute_reply.started":"2022-03-20T09:39:37.070632Z","shell.execute_reply":"2022-03-20T09:39:37.079879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing the Data\n\n* For preparation of data I use code from Alex's awesome kernel [here] (https://www.kaggle.com/shonenkov/training-efficientdet)\n* The data can be split into any number of folds as you want , split is stratified based on number of boxes and source","metadata":{}},{"cell_type":"code","source":"marking = pd.read_csv('../input/global-wheat-detection/train.csv')\n\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:,i] \n    \nmarking.drop(columns=['bbox'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:37.083802Z","iopub.execute_input":"2022-03-20T09:39:37.084562Z","iopub.status.idle":"2022-03-20T09:39:38.131744Z","shell.execute_reply.started":"2022-03-20T09:39:37.084327Z","shell.execute_reply":"2022-03-20T09:39:38.130589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"marking.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:38.133589Z","iopub.execute_input":"2022-03-20T09:39:38.134023Z","iopub.status.idle":"2022-03-20T09:39:38.159373Z","shell.execute_reply.started":"2022-03-20T09:39:38.133976Z","shell.execute_reply":"2022-03-20T09:39:38.158473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_data = marking.groupby('image_id')\nimages = list(map(lambda x: x.split('.')[0], os.listdir('../input/global-wheat-detection/train/')))\n\ndef get_data(img_id):\n    if img_id not in image_data.groups:\n        return dict(image_id=img_id, source='', boxes=list())\n    \n    data  = image_data.get_group(img_id)\n    source = np.unique(data.source.values)\n    assert len(source)==1, 'corrupted data: %s image_id has many sources: %s' %(img_id,source)\n    source=source[0]\n    boxes = data[['x','y','w','h']].values\n    return dict(image_id = img_id, source=source, boxes = boxes)\n\nimage_list = [get_data(img_id) for img_id in images]\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:38.16106Z","iopub.execute_input":"2022-03-20T09:39:38.161597Z","iopub.status.idle":"2022-03-20T09:39:44.222441Z","shell.execute_reply.started":"2022-03-20T09:39:38.161548Z","shell.execute_reply":"2022-03-20T09:39:44.221239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'total number of images: {len(image_list)}, images with bboxes: {len(image_data)}')\nnull_images=[x['image_id'] for x in image_list if len(x['boxes'])==0]\nlen(null_images)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.223918Z","iopub.execute_input":"2022-03-20T09:39:44.224449Z","iopub.status.idle":"2022-03-20T09:39:44.238487Z","shell.execute_reply.started":"2022-03-20T09:39:44.224354Z","shell.execute_reply":"2022-03-20T09:39:44.236781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_fold_index(lst,n_folds):\n    lens = [len(x['boxes']) for x in lst]\n    lens_unique = np.unique(lens)\n    i = np.random.randint(n_folds)\n    fold_indexes = [[] for _ in range(n_folds)]\n    idx = []\n    \n    for _l in lens_unique:\n        idx.extend(np.nonzero(lens==_l)[0].tolist())\n        if len(idx)<n_folds: continue\n        random.shuffle(idx)\n        while len(idx)>= n_folds:\n            fold_indexes[i].append(lst[idx.pop()]['image_id'])\n            i = (i+1) % n_folds\n    while len(idx):\n        fold_indexes[i].append(lst[idx.pop()]['image_id'])\n        i = (i+1) % n_folds\n    \n    return fold_indexes\n    \nsources = np.unique([x['source'] for x in image_list])\nsplitted_image_list = {s:sorted([x for x in image_list if x['source']==s],key=lambda x: len(x['boxes'])) \n                       for s in sources}\nsplitted_image_list = {k: add_fold_index(v,n_folds=n_folds) for k,v in splitted_image_list.items()}\n\nfold_indexes = [[] for _ in range(n_folds)]\nfor k,v in splitted_image_list.items():\n    for i in range(n_folds):\n        fold_indexes[i].extend(v[i])  \n    \nprint([len(v) for v in fold_indexes])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.240797Z","iopub.execute_input":"2022-03-20T09:39:44.24178Z","iopub.status.idle":"2022-03-20T09:39:44.320654Z","shell.execute_reply.started":"2022-03-20T09:39:44.241734Z","shell.execute_reply":"2022-03-20T09:39:44.319386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:\n    plt.figure(figsize=(10,10))\n    for i,img in enumerate(null_images):\n        plt.subplot(7,7,i+1)\n        plt.imshow(plt.imread(f'../input/global-wheat-detection/train/{img}.jpg'))\n        plt.axis('off')\n        plt.axis('tight')\n        plt.axis('equal')\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.322334Z","iopub.execute_input":"2022-03-20T09:39:44.322777Z","iopub.status.idle":"2022-03-20T09:39:44.330899Z","shell.execute_reply.started":"2022-03-20T09:39:44.322732Z","shell.execute_reply":"2022-03-20T09:39:44.329211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations\n\n* As suggested by aleksendra in her kernel ,augentations will play a major role and hence took her up advice and use awesome augmentations , cut-mix and other will be included in future versions","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose(\n        [\n            A.OneOf(\n            [\n                A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),      \n                A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.9)\n            ],\n            p=0.9),         \n            #A.ToGray(p=0.01),         \n            A.HorizontalFlip(p=0.5),         \n            A.VerticalFlip(p=0.5),         \n            A.Resize(height=512, width=512, p=1),      \n            A.Normalize(max_pixel_value=1),\n            #A.Cutout(num_holes=8, max_h_size=32, max_w_size=32, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0)\n        ], \n        p=1.0,         \n        bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n        )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      A.Normalize(max_pixel_value=1),\n                      ToTensorV2(p=1.0),\n                      ], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.334351Z","iopub.execute_input":"2022-03-20T09:39:44.334972Z","iopub.status.idle":"2022-03-20T09:39:44.352596Z","shell.execute_reply.started":"2022-03-20T09:39:44.334924Z","shell.execute_reply":"2022-03-20T09:39:44.351567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset\n\n* I hope you have the video by now , DETR accepts data in coco format which is (x,y,w,h)(for those who do not know there are two formats coco and pascal(smin,ymin,xmax,ymax) which are widely used) . So now we need to prepare data in that format","metadata":{}},{"cell_type":"code","source":"DIR_TRAIN = '../input/global-wheat-detection/train'\nclass WheatDataset(Dataset):\n    def __init__(self,image_list,transforms=None):\n        self.images = image_list\n        self.transforms = transforms\n        self.img_ids = {x['image_id']:i for i,x in enumerate(image_list)}\n        \n    def get_indices(self,img_ids):\n        return [self.img_ids[x] for x in img_ids]\n        \n    def __len__(self) -> int:\n        return len(self.images)\n    \n    def __getitem__(self,index):\n        record = self.images[index]\n        image_id = record['image_id']\n\n        image = cv2.imread(f'{DIR_TRAIN}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        # DETR takes in data in coco format \n        boxes = record['boxes'] \n        \n        labels =  np.zeros(len(boxes), dtype=np.int32)\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image  = sample['image']\n            boxes  = sample['bboxes']\n            labels = sample['labels']\n\n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        ## detr uses center_x,center_y,width,height !!\n        if len(boxes)>0:\n            boxes = np.array(boxes)\n            boxes[:,2:] /= 2\n            boxes[:,:2] += boxes[:,2:]\n        else:\n            boxes = np.zeros((0,4))\n    \n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        \n        return image, target, image_id","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.354942Z","iopub.execute_input":"2022-03-20T09:39:44.356273Z","iopub.status.idle":"2022-03-20T09:39:44.377916Z","shell.execute_reply.started":"2022-03-20T09:39:44.356225Z","shell.execute_reply":"2022-03-20T09:39:44.376794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = WheatDataset(image_list,get_train_transforms())\nvalid_ds = WheatDataset(image_list,get_valid_transforms())","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.380478Z","iopub.execute_input":"2022-03-20T09:39:44.381428Z","iopub.status.idle":"2022-03-20T09:39:44.393512Z","shell.execute_reply.started":"2022-03-20T09:39:44.381347Z","shell.execute_reply":"2022-03-20T09:39:44.392331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_example(image,target,image_id=None):\n    np_image = image.cpu().numpy().transpose((1,2,0))\n    # unnormalize the image\n    np_image = np_image*np.array([0.229, 0.224, 0.225])+np.array([0.485, 0.456, 0.406])\n    #np_image = (np_image*255).astype(np.uint8)\n    target = {k: v.cpu().numpy() for k, v in target.items()} \n    \n    boxes = target['boxes']\n    h,w,_ = np_image.shape\n    boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(boxes,h,w)]\n        \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for box in boxes:\n        cv2.rectangle(np_image,\n                  (box[0]-box[2], box[1]-box[3]),\n                  (box[2]+box[0], box[3]+box[1]),\n                  (220, 0, 0), 1)\n        \n    ax.set_axis_off()\n    ax.imshow(np_image)\n    ax.set_title(image_id)\n    plt.show()\n\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.395715Z","iopub.execute_input":"2022-03-20T09:39:44.396263Z","iopub.status.idle":"2022-03-20T09:39:44.414545Z","shell.execute_reply.started":"2022-03-20T09:39:44.396214Z","shell.execute_reply":"2022-03-20T09:39:44.413482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_example(*train_ds[350])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.41671Z","iopub.execute_input":"2022-03-20T09:39:44.417371Z","iopub.status.idle":"2022-03-20T09:39:44.804779Z","shell.execute_reply.started":"2022-03-20T09:39:44.417325Z","shell.execute_reply":"2022-03-20T09:39:44.803641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\n* Initial DETR model is trained on coco dataset , which has 91 classes + 1 background class , hence we need to modify it to take our own number of classes\n* Also DETR model takes in 100 queries ie ,it outputs total of 100 bboxes for every image , we can very well change that too","metadata":{}},{"cell_type":"code","source":"#torch.hub.load?","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.806266Z","iopub.execute_input":"2022-03-20T09:39:44.806748Z","iopub.status.idle":"2022-03-20T09:39:44.812001Z","shell.execute_reply.started":"2022-03-20T09:39:44.806708Z","shell.execute_reply":"2022-03-20T09:39:44.810837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def DETRModel(num_classes,model_name=model_name):\n    model = torch.hub.load('facebookresearch/detr', model_name, pretrained=False, num_classes=num_classes)\n    def parameter_groups(self):\n        return { 'backbone': [p for n,p in self.named_parameters()\n                              if ('backbone' in n) and p.requires_grad],\n                 'transformer': [p for n,p in self.named_parameters() \n                                 if (('transformer' in n) or ('input_proj' in n)) and p.requires_grad],\n                 'embed': [p for n,p in self.named_parameters()\n                                 if (('class_embed' in n) or ('bbox_embed' in n) or ('query_embed' in n)) \n                           and p.requires_grad]}\n    setattr(type(model),'parameter_groups',parameter_groups)\n    return model\n\nclass DETRModel(nn.Module):\n    def __init__(self,num_classes=1):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        \n        self.model = torch.hub.load('facebookresearch/detr', model_name, pretrained=True)\n        \n        self.out = nn.Linear(in_features=self.model.class_embed.out_features,out_features=num_classes+1)\n        \n    def forward(self,images):\n        d = self.model(images)\n        d['pred_logits'] = self.out(d['pred_logits'])\n        return d\n    \n    def parameter_groups(self):\n        return { \n            'backbone': [p for n,p in self.model.named_parameters()\n                              if ('backbone' in n) and p.requires_grad],\n            'transformer': [p for n,p in self.model.named_parameters() \n                                 if (('transformer' in n) or ('input_proj' in n)) and p.requires_grad],\n            'embed': [p for n,p in self.model.named_parameters()\n                                 if (('class_embed' in n) or ('bbox_embed' in n) or ('query_embed' in n)) \n                           and p.requires_grad],\n            'final': self.out.parameters()\n            }","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.814233Z","iopub.execute_input":"2022-03-20T09:39:44.815275Z","iopub.status.idle":"2022-03-20T09:39:44.980424Z","shell.execute_reply.started":"2022-03-20T09:39:44.815228Z","shell.execute_reply":"2022-03-20T09:39:44.979444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DETRModel()\nmodel.parameter_groups().keys()\n#type(model)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:44.983299Z","iopub.execute_input":"2022-03-20T09:39:44.984434Z","iopub.status.idle":"2022-03-20T09:39:59.382464Z","shell.execute_reply.started":"2022-03-20T09:39:44.984341Z","shell.execute_reply":"2022-03-20T09:39:59.381163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matcher and Bipartite Matching Loss\n\nNow we make use of the unique loss that the model uses and for that we need to define the matcher. DETR calcuates three individual losses :\n* Classification Loss for labels(its weight can be set by loss_ce)\n* Bbox Loss (its weight can be set by loss_bbox)\n* Loss for Background class","metadata":{}},{"cell_type":"code","source":"'''\ncode taken from github repo detr , 'code present in engine.py'\n'''\n\nmatcher = HungarianMatcher(cost_giou=2,cost_class=1,cost_bbox=5)\n\nweight_dict = {'loss_ce': 1, 'loss_bbox': 5 , 'loss_giou': 2}\n\nlosses = ['labels', 'boxes', 'cardinality']","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:59.384195Z","iopub.execute_input":"2022-03-20T09:39:59.384929Z","iopub.status.idle":"2022-03-20T09:39:59.392789Z","shell.execute_reply.started":"2022-03-20T09:39:59.38488Z","shell.execute_reply":"2022-03-20T09:39:59.391517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:59.394573Z","iopub.execute_input":"2022-03-20T09:39:59.39521Z","iopub.status.idle":"2022-03-20T09:39:59.407871Z","shell.execute_reply.started":"2022-03-20T09:39:59.395159Z","shell.execute_reply":"2022-03-20T09:39:59.406806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_fold(fold):\n    \n    train_indexes = train_ds.get_indices([x for i,f in enumerate(fold_indexes) if i!=fold for x in f])\n    valid_indexes = valid_ds.get_indices(fold_indexes[fold])\n    \n    train_data_loader = DataLoader(\n        torch.utils.data.Subset(train_ds,train_indexes),\n        batch_size=BATCH_SIZE,\n        shuffle=True,\n        num_workers=2,\n        collate_fn=collate_fn\n    )\n\n    valid_data_loader = DataLoader(\n        torch.utils.data.Subset(valid_ds,valid_indexes),\n        batch_size=BATCH_SIZE,\n        shuffle=False,\n        num_workers=2,\n        collate_fn=collate_fn\n    )\n    return train_data_loader,valid_data_loader\n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:59.411703Z","iopub.execute_input":"2022-03-20T09:39:59.412312Z","iopub.status.idle":"2022-03-20T09:39:59.422639Z","shell.execute_reply.started":"2022-03-20T09:39:59.412262Z","shell.execute_reply":"2022-03-20T09:39:59.421488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader,valid_loader = get_fold(0)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:59.424501Z","iopub.execute_input":"2022-03-20T09:39:59.425365Z","iopub.status.idle":"2022-03-20T09:39:59.43466Z","shell.execute_reply.started":"2022-03-20T09:39:59.425316Z","shell.execute_reply":"2022-03-20T09:39:59.433635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_iter = iter(valid_loader)\nbatch  = next(valid_iter)\n#batch  = next(valid_iter)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:39:59.436997Z","iopub.execute_input":"2022-03-20T09:39:59.437723Z","iopub.status.idle":"2022-03-20T09:40:00.050695Z","shell.execute_reply.started":"2022-03-20T09:39:59.437599Z","shell.execute_reply":"2022-03-20T09:40:00.048532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images,targets,image_id = batch\ntorch.cat([v['boxes'] for v in targets])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.057848Z","iopub.execute_input":"2022-03-20T09:40:00.060914Z","iopub.status.idle":"2022-03-20T09:40:00.080773Z","shell.execute_reply.started":"2022-03-20T09:40:00.06086Z","shell.execute_reply":"2022-03-20T09:40:00.078494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import util.box_ops  as box_ops\n\ndef challenge_metric(outputs,targets):\n    logits = outputs['pred_logits']\n    boxes  = outputs['pred_boxes']\n    return sum(avg_precision(logit[:,0]-logit[:,1],box,target['boxes'])\n            for logit,box,target in zip(logits,boxes,targets))/len(logits)\n\n    return {target['image_id']:avg_precision(logit[:,0]-logit[:,1],box,target['boxes'])\n            for logit,box,target in zip(logits,boxes,targets)}\n\n\n@torch.no_grad()\ndef avg_precision(logit,pboxes,tboxes,reduce=True):\n    idx = logit.gt(0)\n    if sum(idx)==0 and len(tboxes)==0: \n        return 1 if reduce else [1]*6\n    if sum(idx)>0 and len(tboxes)==0: \n        return 0 if reduce else [0]*6\n    \n    pboxes = pboxes[idx]\n    logit = logit[idx]\n    \n    idx = logit.argsort(descending=True)\n    pboxes=box_ops.box_cxcywh_to_xyxy(pboxes.detach()[idx])\n    tboxes=box_ops.box_cxcywh_to_xyxy(tboxes)\n    \n    iou = box_ops.box_iou(pboxes,tboxes)[0].cpu().numpy()\n    prec = [precision(iou,th) for th in [0.5,0.55,0.6,0.65,0.7,0.75]]\n    if reduce:\n        return sum(prec)/6\n    return prec\n    \n\ndef precision(iou,th):\n    #if iou.shape==(0,0): return 1\n\n    #if min(*iou.shape)==0: return 0\n    tp = 0\n    iou = iou.copy()\n    num_pred,num_gt = iou.shape\n    for i in range(num_pred):\n        _iou = iou[i]\n        n_hits = (_iou>th).sum()\n        if n_hits>0:\n            tp += 1\n            j = np.argmax(_iou)\n            iou[:,j] = 0\n    return tp/(num_pred+num_gt-tp)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.086103Z","iopub.execute_input":"2022-03-20T09:40:00.088772Z","iopub.status.idle":"2022-03-20T09:40:00.126341Z","shell.execute_reply.started":"2022-03-20T09:40:00.088726Z","shell.execute_reply":"2022-03-20T09:40:00.124654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_box(n,scale=1):\n    par = torch.randn((n,4)).mul(scale).sigmoid() \n    max_hw = 2*torch.min(par[:,:2],1-par[:,:2])\n    par[:,2:] = par[:,2:].min(max_hw)\n    return par\n\npboxes = gen_box(50)\nlogit = torch.randn(50)\ntboxes = gen_box(3) \n#iou = \navg_precision(logit,pboxes,tboxes)\n#iou.gt(0.5),iou,pboxes,tboxes","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.134089Z","iopub.execute_input":"2022-03-20T09:40:00.137535Z","iopub.status.idle":"2022-03-20T09:40:00.201288Z","shell.execute_reply.started":"2022-03-20T09:40:00.137484Z","shell.execute_reply":"2022-03-20T09:40:00.199896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function\n\nTraining of DETR is unique and different from FasteRRcnn  and EfficientDET , as we train the criterion as well , the training function can be viewed here : https://github.com/facebookresearch/detr/blob/master/engine.py","metadata":{}},{"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    criterion.train()\n    \n    tk0 = tqdm(data_loader, total=len(data_loader),leave=False)\n    log = None\n    \n    for step, (images, targets, image_ids) in enumerate(tk0):\n        \n        batch_size = len(images)\n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n\n        output = model(images)\n        \n        loss_dict = criterion(output, targets)\n        \n        if log is None:\n            log = {k:AverageMeter() for k in loss_dict}\n            log['total_loss'] = AverageMeter()\n            log['avg_prec'] = AverageMeter()\n            \n        weight_dict = criterion.weight_dict\n        \n        total_loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        \n        optimizer.zero_grad()\n\n        total_loss.backward()\n        \n        if max_norm > 0:\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)\n        \n        optimizer.step()\n        \n        if scheduler is not None:\n            scheduler.step()\n        \n        log['total_loss'].update(total_loss.item(),batch_size)\n        \n        for k,v in loss_dict.items():\n            log[k].update(v.item(),batch_size)\n            \n        log['avg_prec'].update(challenge_metric(output,targets),batch_size)\n            \n        tk0.set_postfix({k:v.avg for k,v in log.items()}) \n        \n    return log","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.205474Z","iopub.execute_input":"2022-03-20T09:40:00.208106Z","iopub.status.idle":"2022-03-20T09:40:00.235829Z","shell.execute_reply.started":"2022-03-20T09:40:00.208062Z","shell.execute_reply":"2022-03-20T09:40:00.234372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eval Function","metadata":{}},{"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    model.eval()\n    criterion.eval()\n    log = None\n    \n    with torch.no_grad():\n        \n        tk0 = tqdm(data_loader, total=len(data_loader),leave=False)\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            \n            batch_size = len(images)\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n        \n            if log is None:\n                log = {k:AverageMeter() for k in loss_dict}\n                log['total_loss'] = AverageMeter()\n                log['avg_prec'] = AverageMeter()\n            \n            for k,v in loss_dict.items():\n                log[k].update(v.item(),batch_size)\n        \n            total_loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            log['total_loss'].update(total_loss.item(),batch_size)\n            log['avg_prec'].update(challenge_metric(output,targets),batch_size)\n            \n            tk0.set_postfix({k:v.avg for k,v in log.items()}) \n    \n    return log #['total_loss']","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.242189Z","iopub.execute_input":"2022-03-20T09:40:00.245542Z","iopub.status.idle":"2022-03-20T09:40:00.273242Z","shell.execute_reply.started":"2022-03-20T09:40:00.245491Z","shell.execute_reply":"2022-03-20T09:40:00.271724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine","metadata":{}},{"cell_type":"code","source":"import json\n\nclass Logger:\n    def __init__(self,filename,format='csv'):\n        self.filename = filename + '.' + format\n        self._log = []\n        self.format = format\n    def save(self,log,epoch=None):\n        log['epoch'] = epoch+1\n        self._log.append(log)\n        if self.format == 'json':\n            with open(self.filename,'w') as f:\n                json.dump(self._log,f)\n        else:\n            pd.DataFrame(self._log).to_csv(self.filename,index=False)\n            \n            \ndef run(fold,epochs=EPOCHS):\n    \n    train_data_loader,valid_data_loader = get_fold(fold)\n    \n    logger = Logger(f'log_{fold}')\n    device = torch.device('cuda')\n    model = DETRModel(num_classes=num_classes)\n    model = model.to(device)\n    criterion = SetCriterion(num_classes, \n                             matcher, weight_dict, \n                             eos_coef = null_class_coef, \n                             losses=losses)\n    \n    criterion = criterion.to(device)\n    \n\n    optimizer = torch.optim.AdamW([{\n        'params': v,\n        'lr': lr_dict.get(k,1)*LR\n    } for k,v in model.parameter_groups().items()], weight_decay=1e-4)\n    \n    best_precision = 0\n    header_printed = False\n    for epoch in range(epochs):\n        train_log = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        valid_log = eval_fn(valid_data_loader, model,criterion, device)\n    \n        log = {k:v.avg for k,v in train_log.items()}\n        log.update({'V/'+k:v.avg for k,v in valid_log.items()})\n        logger.save(log,epoch)\n        keys = sorted(log.keys())\n        \n        if not header_printed:\n            print(' '.join(map(lambda k: f'{k[:8]:8}',keys)))\n            header_printed = True\n        print(' '.join(map(lambda k: f'{log[k]:8.3f}'[:8],keys)))\n        \n        if log['V/avg_prec'] > best_precision:\n            best_precision = log['V/avg_prec']\n            print('Best model found at epoch {}'.format(epoch+1))\n            torch.save(model.state_dict(), f'detr_best_{fold}.pth')\n        ","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.288756Z","iopub.execute_input":"2022-03-20T09:40:00.292071Z","iopub.status.idle":"2022-03-20T09:40:00.328632Z","shell.execute_reply.started":"2022-03-20T09:40:00.292024Z","shell.execute_reply":"2022-03-20T09:40:00.326099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:17:02.98049Z","iopub.execute_input":"2022-03-19T19:17:02.981185Z","iopub.status.idle":"2022-03-19T19:17:03.327026Z","shell.execute_reply.started":"2022-03-19T19:17:02.981134Z","shell.execute_reply":"2022-03-19T19:17:03.325871Z"}}},{"cell_type":"code","source":"run(fold=0,epochs=10)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T09:40:00.342947Z","iopub.execute_input":"2022-03-20T09:40:00.346787Z","iopub.status.idle":"2022-03-20T10:24:55.266308Z","shell.execute_reply.started":"2022-03-20T09:40:00.346725Z","shell.execute_reply":"2022-03-20T10:24:55.264766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample\n\n* I know we might be naive to visualize the model ouput just after one epoch but lets do that and see what are the results like","metadata":{}},{"cell_type":"code","source":"def show_predictions(images,outputs,targets):\n    _,h,w = images[0].shape\n    \n    boxes = targets[0]['boxes'].cpu().numpy() #.astype(np.int32)\n    boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(boxes,h,w)]\n    np_image = images[0].permute(1,2,0).cpu().numpy()\n    np_image = np_image*np.array([0.229, 0.224, 0.225])+np.array([0.485, 0.456, 0.406])\n    \n    #outputs = [{k: v.cpu() for k, v in output.items()}]\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for box in boxes:\n        cv2.rectangle(np_image,\n                  (box[0]-box[2], box[1]-box[3]),\n                  (box[2]+box[0], box[3]+box[1]),\n                  (220, 0, 0), 2)\n    \n    oboxes = outputs['pred_boxes'][0].detach().cpu().numpy()\n    oboxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(oboxes,h,w)]\n    prob   = outputs['pred_logits'][0].softmax(1).detach().cpu().numpy()[:,0]\n    for box,p in zip(oboxes,prob):\n        if p>0.5:\n            color = (0,0,220) if p>0.5 else (0,220,0)\n            cv2.rectangle(np_image,\n                      (box[0]-box[2], box[1]-box[3]),\n                      (box[2]+box[0], box[3]+box[1]),\n                      color, 1)\n    \n    ax.set_axis_off()\n    ax.imshow(np_image)\n    #return images,outputs,targets","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:55.273878Z","iopub.execute_input":"2022-03-20T10:24:55.27428Z","iopub.status.idle":"2022-03-20T10:24:55.296308Z","shell.execute_reply.started":"2022-03-20T10:24:55.274243Z","shell.execute_reply":"2022-03-20T10:24:55.293434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DETRModel(num_classes=num_classes)\nmodel.load_state_dict(torch.load(\"./detr_best_0.pth\"))\nmodel.to(torch.device('cuda'))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:55.298522Z","iopub.execute_input":"2022-03-20T10:24:55.299056Z","iopub.status.idle":"2022-03-20T10:24:56.830643Z","shell.execute_reply.started":"2022-03-20T10:24:55.299008Z","shell.execute_reply":"2022-03-20T10:24:56.829441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader,valid_loader = get_fold(2)\nvalid_iter = iter(valid_loader)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:56.832345Z","iopub.execute_input":"2022-03-20T10:24:56.83303Z","iopub.status.idle":"2022-03-20T10:24:56.985004Z","shell.execute_reply.started":"2022-03-20T10:24:56.832985Z","shell.execute_reply":"2022-03-20T10:24:56.981069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images,targets,image_id = next(valid_iter)\ndev_images = [img.to(torch.device('cuda')) for img in images]\nmodel.eval()\nwith torch.no_grad():\n    outputs = model(dev_images)\noutputs = {k: v.cpu() for k, v in outputs.items()}\nshow_predictions(images,outputs,targets)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:56.996294Z","iopub.execute_input":"2022-03-20T10:24:56.997474Z","iopub.status.idle":"2022-03-20T10:24:58.083932Z","shell.execute_reply.started":"2022-03-20T10:24:56.997408Z","shell.execute_reply":"2022-03-20T10:24:58.082594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess(output):\n    logits  = output['pred_logits'][0]\n    idx     = (logits[:,0]-logits[:,1]).gt(0)\n    return {'pred_logits': logits[idx,0]-logits[idx,1],'pred_boxes':output['pred_boxes'][0][idx]}","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:58.086305Z","iopub.execute_input":"2022-03-20T10:24:58.09264Z","iopub.status.idle":"2022-03-20T10:24:58.109357Z","shell.execute_reply.started":"2022-03-20T10:24:58.092587Z","shell.execute_reply":"2022-03-20T10:24:58.10715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logit = outputs['pred_logits'][0]\nprec = avg_precision(logit[:,0]-logit[:,1],outputs['pred_boxes'][0],targets[0]['boxes'],reduce=False)\nprec,sum(prec)/len(prec),(logit[:,0]-logit[:,1]).gt(0)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:58.114295Z","iopub.execute_input":"2022-03-20T10:24:58.117481Z","iopub.status.idle":"2022-03-20T10:24:58.146594Z","shell.execute_reply.started":"2022-03-20T10:24:58.116578Z","shell.execute_reply":"2022-03-20T10:24:58.144805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs['pred_logits'][0].softmax(1).detach().cpu().numpy()[:,0]\n#len(targets[0]['labels']),len(targets[0]['boxes'])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:24:58.14872Z","iopub.execute_input":"2022-03-20T10:24:58.159752Z","iopub.status.idle":"2022-03-20T10:24:58.176603Z","shell.execute_reply.started":"2022-03-20T10:24:58.159655Z","shell.execute_reply":"2022-03-20T10:24:58.175289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}